You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by iv...@apache.org on 2021/05/13 14:44:42 UTC
[ignite-python-thin-client] branch master updated: IGNITE-14705 Fix
handling collections with binary objects - Fixes #37.
This is an automated email from the ASF dual-hosted git repository.
ivandasch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
The following commit(s) were added to refs/heads/master by this push:
new 746dd13 IGNITE-14705 Fix handling collections with binary objects - Fixes #37.
746dd13 is described below
commit 746dd1315a0a86d8f1c501d80f3a91dfafd0f648
Author: Ivan Daschinsky <iv...@apache.org>
AuthorDate: Thu May 13 17:43:50 2021 +0300
IGNITE-14705 Fix handling collections with binary objects - Fixes #37.
---
pyignite/aio_client.py | 27 ++++++++++----
pyignite/client.py | 27 +++++++++-----
pyignite/datatypes/base.py | 12 +++---
pyignite/datatypes/cache_properties.py | 12 +++---
pyignite/datatypes/complex.py | 66 +++++++++++++--------------------
pyignite/datatypes/expiry_policy.py | 4 +-
pyignite/datatypes/internal.py | 48 ++++++++++++------------
pyignite/datatypes/null_object.py | 18 ++++-----
pyignite/datatypes/primitive.py | 6 +--
pyignite/datatypes/primitive_arrays.py | 14 +++----
pyignite/datatypes/primitive_objects.py | 22 +++++------
pyignite/datatypes/standard.py | 38 +++++++++----------
pyignite/queries/response.py | 29 +++++++--------
pyignite/utils.py | 7 ----
tests/common/test_binary.py | 56 +++++++++++++++++++++++++++-
tests/common/test_datatypes.py | 13 +++++++
tests/common/test_key_value.py | 35 ++++++++++++++---
17 files changed, 260 insertions(+), 174 deletions(-)
diff --git a/pyignite/aio_client.py b/pyignite/aio_client.py
index b0498f7..8c2ca56 100644
--- a/pyignite/aio_client.py
+++ b/pyignite/aio_client.py
@@ -31,7 +31,7 @@ from .datatypes import BinaryObject
from .exceptions import BinaryTypeError, CacheError, ReconnectError, connection_errors
from .queries.query import CacheInfo
from .stream import AioBinaryStream, READ_BACKWARD
-from .utils import cache_id, entity_id, status_to_exception, is_wrapped
+from .utils import cache_id, entity_id, status_to_exception
__all__ = ['AioClient']
@@ -269,11 +269,24 @@ class AioClient(BaseClient):
:return: the result of the Binary Object unwrapping with all other data
left intact.
"""
- if is_wrapped(value):
- blob, offset = value
- with AioBinaryStream(self, blob) as stream:
- data_class = await BinaryObject.parse_async(stream)
- return await BinaryObject.to_python_async(stream.read_ctype(data_class, direction=READ_BACKWARD), self)
+ if isinstance(value, tuple) and len(value) == 2:
+ if type(value[0]) is bytes and type(value[1]) is int:
+ blob, offset = value
+ with AioBinaryStream(self, blob) as stream:
+ data_class = await BinaryObject.parse_async(stream)
+ return await BinaryObject.to_python_async(stream.read_ctype(data_class, direction=READ_BACKWARD),
+ client=self)
+
+ if isinstance(value[0], int):
+ col_type, collection = value
+ if isinstance(collection, list):
+ coros = [self.unwrap_binary(v) for v in collection]
+ return col_type, await asyncio.gather(*coros)
+
+ if isinstance(collection, dict):
+ coros = [asyncio.gather(self.unwrap_binary(k), self.unwrap_binary(v))
+ for k, v in collection.items()]
+ return col_type, dict(await asyncio.gather(*coros))
return value
@status_to_exception(CacheError)
@@ -351,7 +364,7 @@ class AioClient(BaseClient):
key, key_hint = self._get_affinity_key(c_id, key, key_hint)
- hashcode = await key_hint.hashcode_async(key, self)
+ hashcode = await key_hint.hashcode_async(key, client=self)
best_node = self._get_node_by_hashcode(c_id, hashcode, parts)
if best_node:
diff --git a/pyignite/client.py b/pyignite/client.py
index 099b44d..01ee373 100644
--- a/pyignite/client.py
+++ b/pyignite/client.py
@@ -61,7 +61,7 @@ from .exceptions import BinaryTypeError, CacheError, ReconnectError, connection_
from .queries.query import CacheInfo
from .stream import BinaryStream, READ_BACKWARD
from .utils import (
- cache_id, capitalize, entity_id, schema_id, process_delimiter, status_to_exception, is_iterable, is_wrapped,
+ cache_id, capitalize, entity_id, schema_id, process_delimiter, status_to_exception, is_iterable,
get_field_by_id, unsigned
)
from .binary import GenericObjectMeta
@@ -539,17 +539,26 @@ class Client(BaseClient):
def unwrap_binary(self, value: Any) -> Any:
"""
- Detects and recursively unwraps Binary Object.
+ Detects and recursively unwraps Binary Object or collections of BinaryObject.
- :param value: anything that could be a Binary Object,
+ :param value: anything that could be a Binary Object or collection of BinaryObject,
:return: the result of the Binary Object unwrapping with all other data
left intact.
"""
- if is_wrapped(value):
- blob, offset = value
- with BinaryStream(self, blob) as stream:
- data_class = BinaryObject.parse(stream)
- return BinaryObject.to_python(stream.read_ctype(data_class, direction=READ_BACKWARD), self)
+ if isinstance(value, tuple) and len(value) == 2:
+ if type(value[0]) is bytes and type(value[1]) is int:
+ blob, offset = value
+ with BinaryStream(self, blob) as stream:
+ data_class = BinaryObject.parse(stream)
+ return BinaryObject.to_python(stream.read_ctype(data_class, direction=READ_BACKWARD), client=self)
+
+ if isinstance(value[0], int):
+ col_type, collection = value
+ if isinstance(collection, list):
+ return col_type, [self.unwrap_binary(v) for v in collection]
+
+ if isinstance(collection, dict):
+ return col_type, {self.unwrap_binary(k): self.unwrap_binary(v) for k, v in collection.items()}
return value
@status_to_exception(CacheError)
@@ -619,7 +628,7 @@ class Client(BaseClient):
return conn
key, key_hint = self._get_affinity_key(c_id, key, key_hint)
- hashcode = key_hint.hashcode(key, self)
+ hashcode = key_hint.hashcode(key, client=self)
best_node = self._get_node_by_hashcode(c_id, hashcode, parts)
if best_node:
diff --git a/pyignite/datatypes/base.py b/pyignite/datatypes/base.py
index 87b251c..5a4c780 100644
--- a/pyignite/datatypes/base.py
+++ b/pyignite/datatypes/base.py
@@ -48,11 +48,11 @@ class IgniteDataType(metaclass=IgniteDataTypeMeta):
classes, both object and payload varieties.
"""
@classmethod
- async def hashcode_async(cls, value, *args, **kwargs):
- return cls.hashcode(value, *args, **kwargs)
+ async def hashcode_async(cls, value, **kwargs):
+ return cls.hashcode(value, **kwargs)
@classmethod
- def hashcode(cls, value, *args, **kwargs):
+ def hashcode(cls, value, **kwargs):
return 0
@classmethod
@@ -72,9 +72,9 @@ class IgniteDataType(metaclass=IgniteDataTypeMeta):
cls.from_python(stream, value, **kwargs)
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
raise NotImplementedError
@classmethod
- async def to_python_async(cls, ctypes_object, *args, **kwargs):
- return cls.to_python(ctypes_object, *args, **kwargs)
+ async def to_python_async(cls, ctypes_object, **kwargs):
+ return cls.to_python(ctypes_object, **kwargs)
diff --git a/pyignite/datatypes/cache_properties.py b/pyignite/datatypes/cache_properties.py
index a1766f3..49327a3 100644
--- a/pyignite/datatypes/cache_properties.py
+++ b/pyignite/datatypes/cache_properties.py
@@ -117,12 +117,12 @@ class PropBase:
return cls.parse(stream)
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
- return cls.prop_data_class.to_python(ctypes_object.data, *args, **kwargs)
+ def to_python(cls, ctypes_object, **kwargs):
+ return cls.prop_data_class.to_python(ctypes_object.data, **kwargs)
@classmethod
- async def to_python_async(cls, ctypes_object, *args, **kwargs):
- return cls.to_python(ctypes_object, *args, **kwargs)
+ async def to_python_async(cls, ctypes_object, **kwargs):
+ return cls.to_python(ctypes_object, **kwargs)
@classmethod
def from_python(cls, stream, value):
@@ -302,6 +302,6 @@ class AnyProperty(PropBase):
)
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
prop_data_class = prop_map(ctypes_object.prop_code)
- return prop_data_class.to_python(ctypes_object.data, *args, **kwargs)
+ return prop_data_class.to_python(ctypes_object.data, **kwargs)
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index 119c552..cddf743 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -90,22 +90,21 @@ class ObjectArrayObject(Nullable):
)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
result = []
for i in range(ctypes_object.length):
result.append(
AnyDataObject.to_python(
- getattr(ctypes_object, f'element_{i}'),
- *args, **kwargs
+ getattr(ctypes_object, f'element_{i}'), **kwargs
)
)
return ctypes_object.type_id, result
@classmethod
- async def to_python_not_null_async(cls, ctypes_object, *args, **kwargs):
+ async def to_python_not_null_async(cls, ctypes_object, **kwargs):
result = [
await AnyDataObject.to_python_async(
- getattr(ctypes_object, f'element_{i}'), *args, **kwargs
+ getattr(ctypes_object, f'element_{i}'), **kwargs
)
for i in range(ctypes_object.length)]
return ctypes_object.type_id, result
@@ -223,8 +222,6 @@ class CollectionObject(Nullable):
_type_id = TYPE_COL
_header_class = None
type_code = TC_COLLECTION
- pythonic = list
- default = []
@classmethod
def parse_not_null(cls, stream):
@@ -271,7 +268,7 @@ class CollectionObject(Nullable):
@classmethod
def to_python_not_null(cls, ctypes_object, *args, **kwargs):
result = [
- AnyDataObject.to_python(getattr(ctypes_object, f'element_{i}'), *args, **kwargs)
+ AnyDataObject.to_python(getattr(ctypes_object, f'element_{i}'), **kwargs)
for i in range(ctypes_object.length)
]
return ctypes_object.type, result
@@ -279,7 +276,7 @@ class CollectionObject(Nullable):
@classmethod
async def to_python_not_null_async(cls, ctypes_object, *args, **kwargs):
result_coro = [
- AnyDataObject.to_python_async(getattr(ctypes_object, f'element_{i}'), *args, **kwargs)
+ AnyDataObject.to_python_async(getattr(ctypes_object, f'element_{i}'), **kwargs)
for i in range(ctypes_object.length)
]
@@ -361,35 +358,27 @@ class _MapBase:
)
@classmethod
- def _to_python(cls, ctypes_object, *args, **kwargs):
+ def _to_python(cls, ctypes_object, **kwargs):
map_cls = cls.__get_map_class(ctypes_object)
result = map_cls()
for i in range(0, ctypes_object.length << 1, 2):
- k = AnyDataObject.to_python(
- getattr(ctypes_object, f'element_{i}'),
- *args, **kwargs
- )
- v = AnyDataObject.to_python(
- getattr(ctypes_object, f'element_{i + 1}'),
- *args, **kwargs
- )
+ k = AnyDataObject.to_python(getattr(ctypes_object, f'element_{i}'), **kwargs)
+ v = AnyDataObject.to_python(getattr(ctypes_object, f'element_{i + 1}'), **kwargs)
result[k] = v
return result
@classmethod
- async def _to_python_async(cls, ctypes_object, *args, **kwargs):
+ async def _to_python_async(cls, ctypes_object, **kwargs):
map_cls = cls.__get_map_class(ctypes_object)
kv_pairs_coro = [
asyncio.gather(
AnyDataObject.to_python_async(
- getattr(ctypes_object, f'element_{i}'),
- *args, **kwargs
+ getattr(ctypes_object, f'element_{i}'), **kwargs
),
AnyDataObject.to_python_async(
- getattr(ctypes_object, f'element_{i + 1}'),
- *args, **kwargs
+ getattr(ctypes_object, f'element_{i + 1}'), **kwargs
)
) for i in range(0, ctypes_object.length << 1, 2)
]
@@ -449,12 +438,12 @@ class Map(IgniteDataType, _MapBase):
return [('length', ctypes.c_int)], length
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
- return cls._to_python(ctypes_object, *args, **kwargs)
+ def to_python(cls, ctypes_object, **kwargs):
+ return cls._to_python(ctypes_object, **kwargs)
@classmethod
- async def to_python_async(cls, ctypes_object, *args, **kwargs):
- return await cls._to_python_async(ctypes_object, *args, **kwargs)
+ async def to_python_async(cls, ctypes_object, **kwargs):
+ return await cls._to_python_async(ctypes_object, **kwargs)
@classmethod
def from_python(cls, stream, value, type_id=None):
@@ -484,8 +473,6 @@ class MapObject(Nullable, _MapBase):
_type_name = NAME_MAP
_type_id = TYPE_MAP
type_code = TC_MAP
- pythonic = dict
- default = {}
@classmethod
def parse_not_null(cls, stream):
@@ -507,12 +494,12 @@ class MapObject(Nullable, _MapBase):
return fields, length
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
- return ctypes_object.type, cls._to_python(ctypes_object, *args, **kwargs)
+ def to_python_not_null(cls, ctypes_object, **kwargs):
+ return ctypes_object.type, cls._to_python(ctypes_object, **kwargs)
@classmethod
- async def to_python_not_null_async(cls, ctypes_object, *args, **kwargs):
- return ctypes_object.type, await cls._to_python_async(ctypes_object, *args, **kwargs)
+ async def to_python_not_null_async(cls, ctypes_object, **kwargs):
+ return ctypes_object.type, await cls._to_python_async(ctypes_object, **kwargs)
@classmethod
def from_python_not_null(cls, stream, value, **kwargs):
@@ -557,7 +544,7 @@ class BinaryObject(Nullable):
COMPACT_FOOTER = 0x0020
@classmethod
- def hashcode(cls, value: object, client: Optional['Client']) -> int:
+ def hashcode(cls, value: object, client: Optional['Client'] = None) -> int:
# binary objects's hashcode implementation is special in the sense
# that you need to fully serialize the object to calculate
# its hashcode
@@ -568,7 +555,7 @@ class BinaryObject(Nullable):
return value._hashcode
@classmethod
- async def hashcode_async(cls, value: object, client: Optional['AioClient']) -> int:
+ async def hashcode_async(cls, value: object, client: Optional['AioClient'] = None) -> int:
if not value._hashcode and client:
with AioBinaryStream(client) as stream:
await value._from_python_async(stream, save_to_buf=True)
@@ -680,7 +667,7 @@ class BinaryObject(Nullable):
return final_class
@classmethod
- def to_python_not_null(cls, ctypes_object, client: 'Client' = None, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, client: 'Client' = None, **kwargs):
type_id = ctypes_object.type_id
if not client:
raise ParseError(f'Can not query binary type {type_id}')
@@ -692,14 +679,13 @@ class BinaryObject(Nullable):
for field_name, field_type in data_class.schema.items():
setattr(
result, field_name, field_type.to_python(
- getattr(ctypes_object.object_fields, field_name),
- client, *args, **kwargs
+ getattr(ctypes_object.object_fields, field_name), client=client, **kwargs
)
)
return result
@classmethod
- async def to_python_not_null_async(cls, ctypes_object, client: 'AioClient' = None, *args, **kwargs):
+ async def to_python_not_null_async(cls, ctypes_object, client: 'AioClient' = None, **kwargs):
type_id = ctypes_object.type_id
if not client:
raise ParseError(f'Can not query binary type {type_id}')
@@ -711,7 +697,7 @@ class BinaryObject(Nullable):
field_values = await asyncio.gather(
*[
field_type.to_python_async(
- getattr(ctypes_object.object_fields, field_name), client, *args, **kwargs
+ getattr(ctypes_object.object_fields, field_name), client=client, **kwargs
)
for field_name, field_type in data_class.schema.items()
]
diff --git a/pyignite/datatypes/expiry_policy.py b/pyignite/datatypes/expiry_policy.py
index 3572754..d729da5 100644
--- a/pyignite/datatypes/expiry_policy.py
+++ b/pyignite/datatypes/expiry_policy.py
@@ -80,14 +80,14 @@ class ExpiryPolicy:
return cls.parse(stream)
@classmethod
- def to_python(cls, ctypes_object):
+ def to_python(cls, ctypes_object, **kwargs):
if ctypes_object == 0:
return None
return ExpiryPolicy(create=ctypes_object.create, update=ctypes_object.update, access=ctypes_object.access)
@classmethod
- async def to_python_async(cls, ctypes_object):
+ async def to_python_async(cls, ctypes_object, **kwargs):
return cls.to_python(ctypes_object)
@classmethod
diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py
index 9bd1b76..54d72bf 100644
--- a/pyignite/datatypes/internal.py
+++ b/pyignite/datatypes/internal.py
@@ -136,15 +136,15 @@ class Conditional:
return await self.var1.parse_async(stream)
return await self.var2.parse_async(stream)
- def to_python(self, ctypes_object, context, *args, **kwargs):
+ def to_python(self, ctypes_object, context, **kwargs):
if self.predicate2(context):
- return self.var1.to_python(ctypes_object, *args, **kwargs)
- return self.var2.to_python(ctypes_object, *args, **kwargs)
+ return self.var1.to_python(ctypes_object, **kwargs)
+ return self.var2.to_python(ctypes_object, **kwargs)
- async def to_python_async(self, ctypes_object, context, *args, **kwargs):
+ async def to_python_async(self, ctypes_object, context, **kwargs):
if self.predicate2(context):
- return await self.var1.to_python_async(ctypes_object, *args, **kwargs)
- return await self.var2.to_python_async(ctypes_object, *args, **kwargs)
+ return await self.var1.to_python_async(ctypes_object, **kwargs)
+ return await self.var2.to_python_async(ctypes_object, **kwargs)
@attr.s
@@ -192,19 +192,17 @@ class StructArray:
},
)
- def to_python(self, ctypes_object, *args, **kwargs):
+ def to_python(self, ctypes_object, **kwargs):
length = getattr(ctypes_object, 'length', 0)
return [
- Struct(self.following, dict_type=dict).to_python(getattr(ctypes_object, f'element_{i}'),
- *args, **kwargs)
+ Struct(self.following, dict_type=dict).to_python(getattr(ctypes_object, f'element_{i}'), **kwargs)
for i in range(length)
]
- async def to_python_async(self, ctypes_object, *args, **kwargs):
+ async def to_python_async(self, ctypes_object, **kwargs):
length = getattr(ctypes_object, 'length', 0)
result_coro = [
- Struct(self.following, dict_type=dict).to_python_async(getattr(ctypes_object, f'element_{i}'),
- *args, **kwargs)
+ Struct(self.following, dict_type=dict).to_python_async(getattr(ctypes_object, f'element_{i}'), **kwargs)
for i in range(length)
]
return await asyncio.gather(*result_coro)
@@ -284,21 +282,21 @@ class Struct:
},
)
- def to_python(self, ctypes_object, *args, **kwargs) -> Union[dict, OrderedDict]:
+ def to_python(self, ctypes_object, **kwargs) -> Union[dict, OrderedDict]:
result = self.dict_type()
for name, c_type in self.fields:
is_cond = isinstance(c_type, Conditional)
result[name] = c_type.to_python(
getattr(ctypes_object, name),
result,
- *args, **kwargs
+ **kwargs
) if is_cond else c_type.to_python(
getattr(ctypes_object, name),
- *args, **kwargs
+ **kwargs
)
return result
- async def to_python_async(self, ctypes_object, *args, **kwargs) -> Union[dict, OrderedDict]:
+ async def to_python_async(self, ctypes_object, **kwargs) -> Union[dict, OrderedDict]:
result = self.dict_type()
for name, c_type in self.fields:
is_cond = isinstance(c_type, Conditional)
@@ -307,12 +305,12 @@ class Struct:
value = await c_type.to_python_async(
getattr(ctypes_object, name),
result,
- *args, **kwargs
+ **kwargs
)
else:
value = await c_type.to_python_async(
getattr(ctypes_object, name),
- *args, **kwargs
+ **kwargs
)
result[name] = value
return result
@@ -394,14 +392,14 @@ class AnyDataObject:
raise ParseError('Unknown type code: `{}`'.format(type_code))
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
data_class = cls.__data_class_from_ctype(ctypes_object)
- return data_class.to_python(ctypes_object)
+ return data_class.to_python(ctypes_object, **kwargs)
@classmethod
- async def to_python_async(cls, ctypes_object, *args, **kwargs):
+ async def to_python_async(cls, ctypes_object, **kwargs):
data_class = cls.__data_class_from_ctype(ctypes_object)
- return await data_class.to_python_async(ctypes_object)
+ return await data_class.to_python_async(ctypes_object, **kwargs)
@classmethod
def __data_class_from_ctype(cls, ctypes_object):
@@ -580,16 +578,16 @@ class AnyDataArray(AnyDataObject):
)
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
length = getattr(ctypes_object, "length", 0)
return [
- super().to_python(getattr(ctypes_object, f'element_{i}'), *args, **kwargs)
+ super().to_python(getattr(ctypes_object, f'element_{i}'), **kwargs)
for i in range(length)
]
@classmethod
- async def to_python_async(cls, ctypes_object, *args, **kwargs):
+ async def to_python_async(cls, ctypes_object, **kwargs):
length = getattr(ctypes_object, "length", 0)
values = asyncio.gather(
diff --git a/pyignite/datatypes/null_object.py b/pyignite/datatypes/null_object.py
index 8ac47b2..d51e5fb 100644
--- a/pyignite/datatypes/null_object.py
+++ b/pyignite/datatypes/null_object.py
@@ -57,7 +57,7 @@ class Null(IgniteDataType):
return cls.build_c_type()
@classmethod
- def to_python(cls, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
return None
@classmethod
@@ -105,7 +105,7 @@ class Nullable(IgniteDataType):
if value is None:
Null.from_python(stream)
else:
- cls.from_python_not_null(stream, value)
+ cls.from_python_not_null(stream, value, **kwargs)
@classmethod
async def from_python_async(cls, stream, value, **kwargs):
@@ -115,26 +115,26 @@ class Nullable(IgniteDataType):
await cls.from_python_not_null_async(stream, value, **kwargs)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
raise NotImplementedError
@classmethod
- async def to_python_not_null_async(cls, ctypes_object, *args, **kwargs):
- return cls.to_python_not_null(ctypes_object, *args, **kwargs)
+ async def to_python_not_null_async(cls, ctypes_object, **kwargs):
+ return cls.to_python_not_null(ctypes_object, **kwargs)
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
if cls.__is_null(ctypes_object):
return None
- return cls.to_python_not_null(ctypes_object, *args, **kwargs)
+ return cls.to_python_not_null(ctypes_object, **kwargs)
@classmethod
- async def to_python_async(cls, ctypes_object, *args, **kwargs):
+ async def to_python_async(cls, ctypes_object, **kwargs):
if cls.__is_null(ctypes_object):
return None
- return await cls.to_python_not_null_async(ctypes_object, *args, **kwargs)
+ return await cls.to_python_not_null_async(ctypes_object, **kwargs)
@classmethod
def __check_null_input(cls, stream):
diff --git a/pyignite/datatypes/primitive.py b/pyignite/datatypes/primitive.py
index 037f680..2213f3d 100644
--- a/pyignite/datatypes/primitive.py
+++ b/pyignite/datatypes/primitive.py
@@ -52,7 +52,7 @@ class Primitive(IgniteDataType):
return cls.c_type
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
return ctypes_object
@@ -122,7 +122,7 @@ class Char(Primitive):
c_type = ctypes.c_short
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
return ctypes_object.value.to_bytes(
ctypes.sizeof(cls.c_type),
byteorder=PROTOCOL_BYTE_ORDER
@@ -147,7 +147,7 @@ class Bool(Primitive):
c_type = ctypes.c_byte # Use c_byte because c_bool throws endianness conversion error on BE systems.
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
return ctypes_object != 0
@classmethod
diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py
index e1d4289..fcf877c 100644
--- a/pyignite/datatypes/primitive_arrays.py
+++ b/pyignite/datatypes/primitive_arrays.py
@@ -67,7 +67,7 @@ class PrimitiveArray(IgniteDataType):
return c_type
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
return [ctypes_object.data[i] for i in range(ctypes_object.length)]
@classmethod
@@ -88,7 +88,7 @@ class ByteArray(PrimitiveArray):
type_code = TC_BYTE_ARRAY
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
+ def to_python(cls, ctypes_object, **kwargs):
return bytes(ctypes_object.data)
@classmethod
@@ -184,7 +184,7 @@ class PrimitiveArrayObject(Nullable):
return c_type
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return [ctypes_object.data[i] for i in range(ctypes_object.length)]
@classmethod
@@ -206,7 +206,7 @@ class ByteArrayObject(PrimitiveArrayObject):
type_code = TC_BYTE_ARRAY
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return bytes(ctypes_object.data)
@classmethod
@@ -277,8 +277,8 @@ class CharArrayObject(PrimitiveArrayObject):
type_code = TC_CHAR_ARRAY
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
- values = super().to_python_not_null(ctypes_object, *args, **kwargs)
+ def to_python_not_null(cls, ctypes_object, **kwargs):
+ values = super().to_python_not_null(ctypes_object, **kwargs)
return [
v.to_bytes(
ctypes.sizeof(cls.primitive_type.c_type),
@@ -296,5 +296,5 @@ class BoolArrayObject(PrimitiveArrayObject):
type_code = TC_BOOL_ARRAY
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return [ctypes_object.data[i] != 0 for i in range(ctypes_object.length)]
diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py
index 9b23ec9..4e66334 100644
--- a/pyignite/datatypes/primitive_objects.py
+++ b/pyignite/datatypes/primitive_objects.py
@@ -65,7 +65,7 @@ class DataObject(Nullable):
return data_type
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return ctypes_object.value
@classmethod
@@ -89,7 +89,7 @@ class ByteObject(DataObject):
default = 0
@classmethod
- def hashcode(cls, value: int, *args, **kwargs) -> int:
+ def hashcode(cls, value: int, **kwargs) -> int:
return value
@@ -102,7 +102,7 @@ class ShortObject(DataObject):
default = 0
@classmethod
- def hashcode(cls, value: int, *args, **kwargs) -> int:
+ def hashcode(cls, value: int, **kwargs) -> int:
return value
@@ -115,7 +115,7 @@ class IntObject(DataObject):
default = 0
@classmethod
- def hashcode(cls, value: int, *args, **kwargs) -> int:
+ def hashcode(cls, value: int, **kwargs) -> int:
return value
@@ -128,7 +128,7 @@ class LongObject(DataObject):
default = 0
@classmethod
- def hashcode(cls, value: int, *args, **kwargs) -> int:
+ def hashcode(cls, value: int, **kwargs) -> int:
return value ^ (unsigned(value, ctypes.c_ulonglong) >> 32)
@@ -141,7 +141,7 @@ class FloatObject(DataObject):
default = 0.0
@classmethod
- def hashcode(cls, value: float, *args, **kwargs) -> int:
+ def hashcode(cls, value: float, **kwargs) -> int:
return ctypes.cast(
ctypes.pointer(ctypes.c_float(value)),
ctypes.POINTER(ctypes.c_int)
@@ -157,7 +157,7 @@ class DoubleObject(DataObject):
default = 0.0
@classmethod
- def hashcode(cls, value: float, *args, **kwargs) -> int:
+ def hashcode(cls, value: float, **kwargs) -> int:
bits = ctypes.cast(
ctypes.pointer(ctypes.c_double(value)),
ctypes.POINTER(ctypes.c_longlong)
@@ -180,11 +180,11 @@ class CharObject(DataObject):
default = ' '
@classmethod
- def hashcode(cls, value: str, *args, **kwargs) -> int:
+ def hashcode(cls, value: str, **kwargs) -> int:
return ord(value)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
value = ctypes_object.value
return value.to_bytes(
ctypes.sizeof(cls.c_type),
@@ -214,9 +214,9 @@ class BoolObject(DataObject):
default = False
@classmethod
- def hashcode(cls, value: bool, *args, **kwargs) -> int:
+ def hashcode(cls, value: bool, **kwargs) -> int:
return 1231 if value else 1237
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return ctypes_object.value != 0
diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py
index 5657afb..9173daa 100644
--- a/pyignite/datatypes/standard.py
+++ b/pyignite/datatypes/standard.py
@@ -71,7 +71,7 @@ class String(Nullable):
pythonic = str
@classmethod
- def hashcode(cls, value: str, *args, **kwargs) -> int:
+ def hashcode(cls, value: str, **kwargs) -> int:
return hashcode(value)
@classmethod
@@ -101,7 +101,7 @@ class String(Nullable):
return data_type
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
if ctypes_object.length > 0:
return ctypes_object.data.decode(PROTOCOL_STRING_ENCODING)
@@ -132,7 +132,7 @@ class DecimalObject(Nullable):
default = decimal.Decimal('0.00')
@classmethod
- def hashcode(cls, value: decimal.Decimal, *args, **kwargs) -> int:
+ def hashcode(cls, value: decimal.Decimal, **kwargs) -> int:
return decimal_hashcode(value)
@classmethod
@@ -163,7 +163,7 @@ class DecimalObject(Nullable):
return data_type
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
sign = 1 if ctypes_object.data[0] & 0x80 else 0
data = ctypes_object.data[1:]
data.insert(0, ctypes_object.data[0] & 0x7f)
@@ -227,7 +227,7 @@ class UUIDObject(StandardObject):
UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
@classmethod
- def hashcode(cls, value: 'UUID', *args, **kwargs) -> int:
+ def hashcode(cls, value: 'UUID', **kwargs) -> int:
msb = value.int >> 64
lsb = value.int & 0xffffffffffffffff
hilo = msb ^ lsb
@@ -263,7 +263,7 @@ class UUIDObject(StandardObject):
stream.write(data_object)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
uuid_array = bytearray(ctypes_object.value)
return uuid.UUID(
bytes=bytes([uuid_array[i] for i in cls.UUID_BYTE_ORDER])
@@ -289,7 +289,7 @@ class TimestampObject(StandardObject):
default = (datetime(1970, 1, 1), 0)
@classmethod
- def hashcode(cls, value: Tuple[datetime, int], *args, **kwargs) -> int:
+ def hashcode(cls, value: Tuple[datetime, int], **kwargs) -> int:
return datetime_hashcode(int(value[0].timestamp() * 1000))
@classmethod
@@ -323,7 +323,7 @@ class TimestampObject(StandardObject):
stream.write(data_object)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return (
datetime.fromtimestamp(ctypes_object.epoch / 1000),
ctypes_object.fraction
@@ -345,7 +345,7 @@ class DateObject(StandardObject):
default = datetime(1970, 1, 1)
@classmethod
- def hashcode(cls, value: datetime, *args, **kwargs) -> int:
+ def hashcode(cls, value: datetime, **kwargs) -> int:
return datetime_hashcode(int(value.timestamp() * 1000))
@classmethod
@@ -379,7 +379,7 @@ class DateObject(StandardObject):
stream.write(data_object)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return datetime.fromtimestamp(ctypes_object.epoch / 1000)
@@ -397,7 +397,7 @@ class TimeObject(StandardObject):
default = timedelta()
@classmethod
- def hashcode(cls, value: timedelta, *args, **kwargs) -> int:
+ def hashcode(cls, value: timedelta, **kwargs) -> int:
return datetime_hashcode(int(value.total_seconds() * 1000))
@classmethod
@@ -429,7 +429,7 @@ class TimeObject(StandardObject):
stream.write(data_object)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return timedelta(milliseconds=ctypes_object.value)
@@ -476,7 +476,7 @@ class EnumObject(StandardObject):
stream.write(data_object)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
+ def to_python_not_null(cls, ctypes_object, **kwargs):
return ctypes_object.type_id, ctypes_object.ordinal
@@ -570,8 +570,8 @@ class StandardArray(IgniteDataType, _StandardArrayBase):
cls._from_python(stream, value, **kwargs)
@classmethod
- def to_python(cls, ctypes_object, *args, **kwargs):
- return cls._to_python(ctypes_object, *args, **kwargs)
+ def to_python(cls, ctypes_object, **kwargs):
+ return cls._to_python(ctypes_object, **kwargs)
class StringArray(StandardArray):
@@ -660,8 +660,8 @@ class StandardArrayObject(Nullable, _StandardArrayBase):
cls._from_python(stream, value, **kwargs)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
- return cls._to_python(ctypes_object, *args, **kwargs)
+ def to_python_not_null(cls, ctypes_object, **kwargs):
+ return cls._to_python(ctypes_object, **kwargs)
class StringArrayObject(StandardArrayObject):
@@ -759,8 +759,8 @@ class EnumArrayObject(StandardArrayObject):
super().from_python_not_null(stream, value, type_id=type_id)
@classmethod
- def to_python_not_null(cls, ctypes_object, *args, **kwargs):
- return ctypes_object.type_id, cls._to_python(ctypes_object, *args, **kwargs)
+ def to_python_not_null(cls, ctypes_object, **kwargs):
+ return ctypes_object.type_id, cls._to_python(ctypes_object, **kwargs)
class BinaryEnumArrayObject(EnumArrayObject):
diff --git a/pyignite/queries/response.py b/pyignite/queries/response.py
index c0311ec..11e71a7 100644
--- a/pyignite/queries/response.py
+++ b/pyignite/queries/response.py
@@ -128,25 +128,22 @@ class Response:
c_type = await ignite_type.parse_async(stream)
fields.append((name, c_type))
- def to_python(self, ctypes_object, *args, **kwargs):
+ def to_python(self, ctypes_object, **kwargs):
if not self.following:
return None
result = OrderedDict()
for name, c_type in self.following:
- result[name] = c_type.to_python(
- getattr(ctypes_object, name),
- *args, **kwargs
- )
+ result[name] = c_type.to_python(getattr(ctypes_object, name), **kwargs)
return result
- async def to_python_async(self, ctypes_object, *args, **kwargs):
+ async def to_python_async(self, ctypes_object, **kwargs):
if not self.following:
return None
values = await asyncio.gather(
- *[c_type.to_python_async(getattr(ctypes_object, name), *args, **kwargs) for name, c_type in self.following]
+ *[c_type.to_python_async(getattr(ctypes_object, name), **kwargs) for name, c_type in self.following]
)
return OrderedDict([(name, values[i]) for i, (name, _) in enumerate(self.following)])
@@ -239,9 +236,9 @@ class SQLResponse(Response):
('more', ctypes.c_byte),
]
- def to_python(self, ctypes_object, *args, **kwargs):
+ def to_python(self, ctypes_object, **kwargs):
if getattr(ctypes_object, 'status_code', 0) == 0:
- result = self.__to_python_result_header(ctypes_object, *args, **kwargs)
+ result = self.__to_python_result_header(ctypes_object, **kwargs)
for row_item in ctypes_object.data._fields_:
row_name = row_item[0]
@@ -250,13 +247,13 @@ class SQLResponse(Response):
for col_item in row_object._fields_:
col_name = col_item[0]
col_object = getattr(row_object, col_name)
- row.append(AnyDataObject.to_python(col_object, *args, **kwargs))
+ row.append(AnyDataObject.to_python(col_object, **kwargs))
result['data'].append(row)
return result
- async def to_python_async(self, ctypes_object, *args, **kwargs):
+ async def to_python_async(self, ctypes_object, **kwargs):
if getattr(ctypes_object, 'status_code', 0) == 0:
- result = self.__to_python_result_header(ctypes_object, *args, **kwargs)
+ result = self.__to_python_result_header(ctypes_object, **kwargs)
data_coro = []
for row_item in ctypes_object.data._fields_:
@@ -266,7 +263,7 @@ class SQLResponse(Response):
for col_item in row_object._fields_:
col_name = col_item[0]
col_object = getattr(row_object, col_name)
- row_coro.append(AnyDataObject.to_python_async(col_object, *args, **kwargs))
+ row_coro.append(AnyDataObject.to_python_async(col_object, **kwargs))
data_coro.append(asyncio.gather(*row_coro))
@@ -328,7 +325,7 @@ class BinaryTypeResponse(Response):
return type_exists
- def to_python(self, ctypes_object, *args, **kwargs):
+ def to_python(self, ctypes_object, **kwargs):
if getattr(ctypes_object, 'status_code', 0) == 0:
result = {
'type_exists': Bool.to_python(ctypes_object.type_exists)
@@ -349,5 +346,5 @@ class BinaryTypeResponse(Response):
}
return result
- async def to_python_async(self, ctypes_object, *args, **kwargs):
- return self.to_python(ctypes_object, *args, **kwargs)
+ async def to_python_async(self, ctypes_object, **kwargs):
+ return self.to_python(ctypes_object, **kwargs)
diff --git a/pyignite/utils.py b/pyignite/utils.py
index 975f414..427cceb 100644
--- a/pyignite/utils.py
+++ b/pyignite/utils.py
@@ -69,13 +69,6 @@ def is_hinted(value):
return isinstance(value, tuple) and len(value) == 2 and issubclass(value[1], IgniteDataType)
-def is_wrapped(value: Any) -> bool:
- """
- Check if a value is of WrappedDataObject type.
- """
- return type(value) is tuple and len(value) == 2 and type(value[0]) is bytes and type(value[1]) is int
-
-
def int_overflow(value: int) -> int:
"""
Simulates 32bit integer overflow.
diff --git a/tests/common/test_binary.py b/tests/common/test_binary.py
index c94c4d5..449709e 100644
--- a/tests/common/test_binary.py
+++ b/tests/common/test_binary.py
@@ -451,13 +451,13 @@ def complex_objects():
def test_complex_object_hash(client, complex_objects):
for obj, hash in complex_objects:
- assert hash == BinaryObject.hashcode(obj, client)
+ assert hash == BinaryObject.hashcode(obj, client=client)
@pytest.mark.asyncio
async def test_complex_object_hash_async(async_client, complex_objects):
for obj, hash in complex_objects:
- assert hash == await BinaryObject.hashcode_async(obj, async_client)
+ assert hash == await BinaryObject.hashcode_async(obj, client=async_client)
def camel_to_snake(name):
@@ -504,3 +504,55 @@ async def test_complex_object_null_fields_async(async_cache, null_fields_object)
"""
await async_cache.put(1, null_fields_object)
assert await async_cache.get(1) == null_fields_object, 'Objects mismatch'
+
+
+def test_object_with_collections_of_binary_objects(cache):
+ __check_object_with_collections_of_binary_objects(cache)
+
+
+@pytest.mark.asyncio
+async def test_object_with_collections_of_binary_objects_async(async_cache):
+ await __check_object_with_collections_of_binary_objects(async_cache)
+
+
+def __check_object_with_collections_of_binary_objects(cache):
+ class Container(
+ metaclass=GenericObjectMeta,
+ schema={
+ 'id': IntObject,
+ 'collection': CollectionObject,
+ 'array': ObjectArrayObject,
+ 'map': MapObject
+ }
+ ):
+ pass
+
+ class Value(
+ metaclass=GenericObjectMeta,
+ schema={
+ 'id': IntObject,
+ 'name': String
+ }
+ ):
+ pass
+
+ def fixtures():
+ map_obj = (MapObject.HASH_MAP, {i: Value(i, f'val_{i}') for i in range(10)})
+ col_obj = (CollectionObject.ARR_LIST, [Value(i, f'val_{i}') for i in range(10)])
+ arr_obj = (ObjectArrayObject.OBJECT, [Value(i, f'val_{i}') for i in range(10)])
+ return [
+ Container(1, map=map_obj, collection=col_obj, array=arr_obj),
+ Container(2), # Check if collections are not set
+ ]
+
+ async def inner_async():
+ for i, val in enumerate(fixtures()):
+ await cache.put(i, val)
+ assert await cache.get(i) == val
+
+ def inner():
+ for i, val in enumerate(fixtures()):
+ cache.put(i, val)
+ assert cache.get(i) == val
+
+ return inner_async() if isinstance(cache, AioCache) else inner()
diff --git a/tests/common/test_datatypes.py b/tests/common/test_datatypes.py
index 6771f94..ebbafb6 100644
--- a/tests/common/test_datatypes.py
+++ b/tests/common/test_datatypes.py
@@ -20,6 +20,7 @@ import decimal
import pytest
import uuid
+from pyignite import GenericObjectMeta
from pyignite.datatypes import (
ByteObject, IntObject, FloatObject, CharObject, ShortObject, BoolObject, ByteArrayObject, IntArrayObject,
ShortArrayObject, FloatArrayObject, BoolArrayObject, CharArrayObject, TimestampObject, String, BinaryEnumObject,
@@ -27,6 +28,17 @@ from pyignite.datatypes import (
)
from pyignite.utils import unsigned
+
+class Value(
+ metaclass=GenericObjectMeta,
+ schema={
+ 'id': IntObject,
+ 'name': String,
+ }
+):
+ pass
+
+
put_get_data_params = [
# integers
(42, None),
@@ -124,6 +136,7 @@ put_get_data_params = [
# object array
((ObjectArrayObject.OBJECT, [1, 2, decimal.Decimal('3'), bytearray(b'\x10\x20')]), ObjectArrayObject),
+ ((ObjectArrayObject.OBJECT, [Value(id=i, name=f'val_{i}') for i in range(10)]), ObjectArrayObject),
# collection
((CollectionObject.LINKED_LIST, [1, 2, 3]), None),
diff --git a/tests/common/test_key_value.py b/tests/common/test_key_value.py
index b03bec2..e26d373 100644
--- a/tests/common/test_key_value.py
+++ b/tests/common/test_key_value.py
@@ -17,7 +17,8 @@ from datetime import datetime
import pytest
-from pyignite.datatypes import CollectionObject, IntObject, MapObject, TimestampObject
+from pyignite import GenericObjectMeta
+from pyignite.datatypes import CollectionObject, IntObject, MapObject, TimestampObject, String
def test_put_get(cache):
@@ -352,16 +353,35 @@ async def test_cache_get_size_async(async_cache):
assert await async_cache.get_size() == 1
+class Value(
+ metaclass=GenericObjectMeta,
+ schema={
+ 'id': IntObject,
+ 'name': String,
+ }
+):
+ pass
+
+
collection_params = [
[
'simple',
- (1, [(123, IntObject), 678, None, 55.2, ((datetime(year=1996, month=3, day=1), 0), TimestampObject)]),
- (1, [123, 678, None, 55.2, (datetime(year=1996, month=3, day=1), 0)])
+ (CollectionObject.ARR_LIST, [
+ (123, IntObject), 678, None, 55.2, ((datetime(year=1996, month=3, day=1), 0), TimestampObject)
+ ]),
+ (CollectionObject.ARR_LIST, [123, 678, None, 55.2, (datetime(year=1996, month=3, day=1), 0)])
],
[
'nested',
- (1, [123, ((1, [456, 'inner_test_string', 789]), CollectionObject), 'outer_test_string']),
- (1, [123, (1, [456, 'inner_test_string', 789]), 'outer_test_string'])
+ (CollectionObject.ARR_LIST, [
+ 123, ((1, [456, 'inner_test_string', 789]), CollectionObject), 'outer_test_string'
+ ]),
+ (CollectionObject.ARR_LIST, [123, (1, [456, 'inner_test_string', 789]), 'outer_test_string'])
+ ],
+ [
+ 'binary',
+ (CollectionObject.ARR_LIST, [Value(id=i, name=f'val_{i}') for i in range(0, 10)]),
+ (CollectionObject.ARR_LIST, [Value(id=i, name=f'val_{i}') for i in range(0, 10)]),
],
[
'hash_map',
@@ -403,6 +423,11 @@ collection_params = [
}
)
],
+ [
+ 'binary_map',
+ (MapObject.HASH_MAP, {i: Value(id=i, name=f"val_{i}") for i in range(10)}),
+ (MapObject.HASH_MAP, {i: Value(id=i, name=f"val_{i}") for i in range(10)})
+ ]
]