You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by is...@apache.org on 2021/02/02 09:20:47 UTC
[ignite-python-thin-client] branch ignite-13863 created (now
bf76b27)
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a change to branch ignite-13863
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git.
at bf76b27 IGNITE-13863: Minor fix
This branch includes the following new commits:
new 999ea3f IGNITE-13863: Fix Null reading and writing
new bf76b27 IGNITE-13863: Minor fix
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite-python-thin-client] 02/02: IGNITE-13863: Minor fix
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch ignite-13863
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
commit bf76b27d30c13b6e286c6f38e84973bddb6807b1
Author: Igor Sapego <ig...@gmail.com>
AuthorDate: Thu Jan 28 01:19:27 2021 +0300
IGNITE-13863: Minor fix
---
pyignite/datatypes/complex.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index 2d4de53..7fa6bc2 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -454,10 +454,10 @@ class MapObject(Map):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- type = getattr(ctype_object, "type", None)
- if type is None:
+ typ = getattr(ctype_object, "type", None)
+ if typ is None:
return None
- return ctype_object.type, super().to_python(
+ return typ, super().to_python(
ctype_object, *args, **kwargs
)
[ignite-python-thin-client] 01/02: IGNITE-13863: Fix Null reading
and writing
Posted by is...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch ignite-13863
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
commit 999ea3f4534c47269bd3111acc00a909641eba58
Author: Igor Sapego <ig...@gmail.com>
AuthorDate: Wed Dec 2 18:26:44 2020 +0300
IGNITE-13863: Fix Null reading and writing
---
pyignite/datatypes/complex.py | 67 +++++++++++++++++---
pyignite/datatypes/internal.py | 5 +-
pyignite/datatypes/primitive_arrays.py | 33 ++++++++--
pyignite/datatypes/primitive_objects.py | 23 +++++--
pyignite/datatypes/standard.py | 28 ++++++---
tests/test_binary.py | 106 +++++++++++++++++++++++++++++++-
6 files changed, 233 insertions(+), 29 deletions(-)
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index ad2a770..2d4de53 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -20,11 +20,13 @@ from typing import Iterable, Dict
from pyignite.constants import *
from pyignite.exceptions import ParseError
+
from .base import IgniteDataType
from .internal import AnyDataObject, infer_from_python
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -68,8 +70,13 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -91,7 +98,10 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -102,6 +112,9 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -150,8 +163,13 @@ class WrappedDataObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
@@ -243,8 +261,13 @@ class CollectionObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -266,7 +289,10 @@ class CollectionObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -277,6 +303,9 @@ class CollectionObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -330,8 +359,13 @@ class Map(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -420,12 +454,18 @@ class MapObject(Map):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
+ type = getattr(ctype_object, "type", None)
+ if type is None:
+ return None
return ctype_object.type, super().to_python(
ctype_object, *args, **kwargs
)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_id, value = value
return super().from_python(value, type_id)
@@ -539,9 +579,13 @@ class BinaryObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
from pyignite.datatypes import Struct
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
# ignore full schema, always retrieve fields' types and order
@@ -572,14 +616,17 @@ class BinaryObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs):
+ type_id = getattr(ctype_object, "type_id", None)
+ if type_id is None:
+ return None
if not client:
raise ParseError(
- 'Can not query binary type {}'.format(ctype_object.type_id)
+ 'Can not query binary type {}'.format(type_id)
)
data_class = client.query_binary_type(
- ctype_object.type_id,
+ type_id,
ctype_object.schema_id
)
result = data_class()
@@ -596,6 +643,8 @@ class BinaryObject(IgniteDataType):
@classmethod
def from_python(cls, value: object):
+ if value is None:
+ return Null.from_python()
if getattr(value, '_buffer', None) is None:
client = cls.find_client()
diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py
index 9f23ec6..23b9cc4 100644
--- a/pyignite/datatypes/internal.py
+++ b/pyignite/datatypes/internal.py
@@ -479,7 +479,10 @@ class AnyDataArray(AnyDataObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
super().to_python(
getattr(ctype_object, 'element_{}'.format(i)),
diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py
index 3763b96..1b41728 100644
--- a/pyignite/datatypes/primitive_arrays.py
+++ b/pyignite/datatypes/primitive_arrays.py
@@ -17,6 +17,7 @@ import ctypes
from typing import Any
from pyignite.constants import *
+from . import Null
from .base import IgniteDataType
from .primitive import *
from .type_codes import *
@@ -61,8 +62,13 @@ class PrimitiveArray(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header_class()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
cls.__name__,
@@ -82,12 +88,18 @@ class PrimitiveArray(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(ctype_object.data[i])
return result
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
@@ -112,7 +124,10 @@ class ByteArray(PrimitiveArray):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return bytearray(ctype_object.data)
+ data = getattr(ctype_object, "data", None)
+ if data is None:
+ return None
+ return bytearray(data)
@classmethod
def from_python(cls, value):
@@ -210,6 +225,9 @@ class ByteArrayObject(PrimitiveArrayObject):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
header_class = cls.build_header_class()
header = header_class()
header.type_code = int.from_bytes(
@@ -282,6 +300,8 @@ class CharArrayObject(PrimitiveArrayObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
values = super().to_python(ctype_object, *args, **kwargs)
+ if values is None:
+ return None
return [
v.to_bytes(
ctypes.sizeof(cls.primitive_type.c_type),
@@ -302,7 +322,10 @@ class BoolArrayObject(PrimitiveArrayObject):
def to_python(cls, ctype_object, *args, **kwargs):
if not ctype_object:
return None
- result = [False] * ctype_object.length
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ result = [False] * length
+ for i in range(length):
result[i] = ctype_object.data[i] != 0
return result
diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py
index 033ac9e..53f12d2 100644
--- a/pyignite/datatypes/primitive_objects.py
+++ b/pyignite/datatypes/primitive_objects.py
@@ -17,10 +17,12 @@ import ctypes
from pyignite.constants import *
from pyignite.utils import unsigned
+
from .base import IgniteDataType
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -60,16 +62,21 @@ class DataObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
data_type = cls.build_c_type()
- buffer = client.recv(ctypes.sizeof(data_type))
+ buffer = tc_type + client.recv(ctypes.sizeof(data_type) - len(tc_type))
return data_type, buffer
@staticmethod
def to_python(ctype_object, *args, **kwargs):
- return ctype_object.value
+ return getattr(ctype_object, "value", None)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
@@ -185,13 +192,18 @@ class CharObject(DataObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.value.to_bytes(
+ value = getattr(ctype_object, "value", None)
+ if value is None:
+ return None
+ return value.to_bytes(
ctypes.sizeof(cls.c_type),
byteorder=PROTOCOL_BYTE_ORDER
).decode(PROTOCOL_CHAR_ENCODING)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
if type(value) is str:
value = value.encode(PROTOCOL_CHAR_ENCODING)
# assuming either a bytes or an integer
@@ -218,5 +230,8 @@ class BoolObject(DataObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.value != 0
+ value = getattr(ctype_object, "value", None)
+ if value is None:
+ return None
+ return value != 0
diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py
index c65cae4..0f16735 100644
--- a/pyignite/datatypes/standard.py
+++ b/pyignite/datatypes/standard.py
@@ -276,8 +276,6 @@ class UUIDObject(StandardObject):
UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
- UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
-
@staticmethod
def hashcode(value: 'UUID', *args, **kwargs) -> int:
msb = value.int >> 64
@@ -303,6 +301,9 @@ class UUIDObject(StandardObject):
@classmethod
def from_python(cls, value: uuid.UUID):
+ if value is None:
+ return Null.from_python()
+
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
@@ -548,8 +549,6 @@ class EnumObject(StandardObject):
cls.type_code,
byteorder=PROTOCOL_BYTE_ORDER
)
- if value is None:
- return Null.from_python(value)
data_object.type_id, data_object.ordinal = value
return bytes(data_object)
@@ -601,8 +600,13 @@ class StandardArray(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header_class()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
for i in range(header.length):
@@ -623,7 +627,10 @@ class StandardArray(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
cls.standard_type.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -634,6 +641,8 @@ class StandardArray(IgniteDataType):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
@@ -796,6 +805,9 @@ class EnumArrayObject(StandardArrayObject):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_id, value = value
header_class = cls.build_header_class()
header = header_class()
@@ -815,7 +827,9 @@ class EnumArrayObject(StandardArrayObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- type_id = ctype_object.type_id
+ type_id = getattr(ctype_object, "type_id", None)
+ if type_id is None:
+ return None
return type_id, super().to_python(ctype_object, *args, **kwargs)
diff --git a/tests/test_binary.py b/tests/test_binary.py
index 4c45afb..9e798fa 100644
--- a/tests/test_binary.py
+++ b/tests/test_binary.py
@@ -18,7 +18,12 @@ from decimal import Decimal
from pyignite import GenericObjectMeta
from pyignite.datatypes import (
- BinaryObject, BoolObject, IntObject, DecimalObject, LongObject, String,
+ BinaryObject, BoolObject, IntObject, DecimalObject, LongObject, String, ByteObject, ShortObject, FloatObject,
+ DoubleObject, CharObject, UUIDObject, DateObject, TimestampObject, TimeObject, EnumObject, BinaryEnumObject,
+ ByteArrayObject, ShortArrayObject, IntArrayObject, LongArrayObject, FloatArrayObject, DoubleArrayObject,
+ CharArrayObject, BoolArrayObject, UUIDArrayObject, DateArrayObject, TimestampArrayObject, TimeArrayObject,
+ EnumArrayObject, StringArrayObject, DecimalArrayObject, ObjectArrayObject, CollectionObject, MapObject,
+ WrappedDataObject,
)
from pyignite.datatypes.prop_codes import *
@@ -308,8 +313,8 @@ def test_complex_object_names(client):
def test_complex_object_hash(client):
"""
- Test that Python client correctly calculates hash of the binary
- object that contains negative bytes.
+ Test that Python client correctly calculates hash of the binary object that
+ contains negative bytes.
"""
class Internal(
metaclass=GenericObjectMeta,
@@ -355,3 +360,98 @@ def test_complex_object_hash(client):
hash_utf8 = BinaryObject.hashcode(obj_utf8, client=client)
assert hash_utf8 == -1945378474, 'Invalid hashcode value for object with UTF-8 strings'
+
+
+def test_complex_object_null_fields(client):
+ """
+ Test that Python client can correctly write and read binary object that
+ contains null fields.
+ """
+ class AllTypesObject(
+ metaclass=GenericObjectMeta,
+ type_name='TestObject',
+ schema=OrderedDict([
+ ('byteField', ByteObject),
+ ('shortField', ShortObject),
+ ('intField', IntObject),
+ ('longField', LongObject),
+ ('floatField', FloatObject),
+ ('doubleField', DoubleObject),
+ ('charField', CharObject),
+ ('boolField', BoolObject),
+ ('uuidField', UUIDObject),
+ ('dateField', DateObject),
+ ('timestampField', TimestampObject),
+ ('timeField', TimeObject),
+ ('enumField', EnumObject),
+ ('binaryEnumField', BinaryEnumObject),
+ ('byteArrayField', ByteArrayObject),
+ ('shortArrayField', ShortArrayObject),
+ ('intArrayField', IntArrayObject),
+ ('longArrayField', LongArrayObject),
+ ('floatArrayField', FloatArrayObject),
+ ('doubleArrayField', DoubleArrayObject),
+ ('charArrayField', CharArrayObject),
+ ('boolArrayField', BoolArrayObject),
+ ('uuidArrayField', UUIDArrayObject),
+ ('dateArrayField', DateArrayObject),
+ ('timestampArrayField', TimestampArrayObject),
+ ('timeArrayField', TimeArrayObject),
+ ('enumArrayField', EnumArrayObject),
+ ('stringField', String),
+ ('stringArrayField', StringArrayObject),
+ ('decimalField', DecimalObject),
+ ('decimalArrayField', DecimalArrayObject),
+ ('objectArrayField', ObjectArrayObject),
+ ('collectionField', CollectionObject),
+ ('mapField', MapObject),
+ ('binaryObjectField', BinaryObject),
+ ])
+ ):
+ pass
+
+ key = 42
+ null_fields_value = AllTypesObject()
+
+ null_fields_value.byteField = None
+ null_fields_value.shortField = None
+ null_fields_value.intField = 10
+ null_fields_value.longField = None
+ null_fields_value.floatField = None
+ null_fields_value.doubleField = None
+ null_fields_value.charField = None
+ null_fields_value.boolField = None
+ null_fields_value.uuidField = None
+ null_fields_value.dateField = None
+ null_fields_value.timestampField = None
+ null_fields_value.timeField = None
+ null_fields_value.enumField = None
+ null_fields_value.binaryEnumField = None
+ null_fields_value.byteArrayField = None
+ null_fields_value.shortArrayField = None
+ null_fields_value.intArrayField = None
+ null_fields_value.longArrayField = None
+ null_fields_value.floatArrayField = None
+ null_fields_value.doubleArrayField = None
+ null_fields_value.charArrayField = None
+ null_fields_value.boolArrayField = None
+ null_fields_value.uuidArrayField = None
+ null_fields_value.dateArrayField = None
+ null_fields_value.timestampArrayField = None
+ null_fields_value.timeArrayField = None
+ null_fields_value.enumArrayField = None
+ null_fields_value.stringField = None
+ null_fields_value.stringArrayField = None
+ null_fields_value.decimalField = None
+ null_fields_value.decimalArrayField = None
+ null_fields_value.objectArrayField = None
+ null_fields_value.collectionField = None
+ null_fields_value.mapField = None
+ null_fields_value.binaryObjectField = None
+
+ cache = client.get_or_create_cache('all_types_test_cache')
+ cache.put(key, null_fields_value)
+
+ got_obj = cache.get(key)
+
+ assert got_obj == null_fields_value, 'Objects mismatch'