You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by is...@apache.org on 2021/02/02 09:20:48 UTC
[ignite-python-thin-client] 01/02: IGNITE-13863: Fix Null reading
and writing
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch ignite-13863
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
commit 999ea3f4534c47269bd3111acc00a909641eba58
Author: Igor Sapego <ig...@gmail.com>
AuthorDate: Wed Dec 2 18:26:44 2020 +0300
IGNITE-13863: Fix Null reading and writing
---
pyignite/datatypes/complex.py | 67 +++++++++++++++++---
pyignite/datatypes/internal.py | 5 +-
pyignite/datatypes/primitive_arrays.py | 33 ++++++++--
pyignite/datatypes/primitive_objects.py | 23 +++++--
pyignite/datatypes/standard.py | 28 ++++++---
tests/test_binary.py | 106 +++++++++++++++++++++++++++++++-
6 files changed, 233 insertions(+), 29 deletions(-)
diff --git a/pyignite/datatypes/complex.py b/pyignite/datatypes/complex.py
index ad2a770..2d4de53 100644
--- a/pyignite/datatypes/complex.py
+++ b/pyignite/datatypes/complex.py
@@ -20,11 +20,13 @@ from typing import Iterable, Dict
from pyignite.constants import *
from pyignite.exceptions import ParseError
+
from .base import IgniteDataType
from .internal import AnyDataObject, infer_from_python
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -68,8 +70,13 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -91,7 +98,10 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -102,6 +112,9 @@ class ObjectArrayObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -150,8 +163,13 @@ class WrappedDataObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
@@ -243,8 +261,13 @@ class CollectionObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -266,7 +289,10 @@ class CollectionObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
AnyDataObject.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -277,6 +303,9 @@ class CollectionObject(IgniteDataType):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_or_id, value = value
header_class = cls.build_header()
header = header_class()
@@ -330,8 +359,13 @@ class Map(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
@@ -420,12 +454,18 @@ class MapObject(Map):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
+ type = getattr(ctype_object, "type", None)
+ if type is None:
+ return None
return ctype_object.type, super().to_python(
ctype_object, *args, **kwargs
)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_id, value = value
return super().from_python(value, type_id)
@@ -539,9 +579,13 @@ class BinaryObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
from pyignite.datatypes import Struct
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
header_class = cls.build_header()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
# ignore full schema, always retrieve fields' types and order
@@ -572,14 +616,17 @@ class BinaryObject(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, client: 'Client' = None, *args, **kwargs):
+ type_id = getattr(ctype_object, "type_id", None)
+ if type_id is None:
+ return None
if not client:
raise ParseError(
- 'Can not query binary type {}'.format(ctype_object.type_id)
+ 'Can not query binary type {}'.format(type_id)
)
data_class = client.query_binary_type(
- ctype_object.type_id,
+ type_id,
ctype_object.schema_id
)
result = data_class()
@@ -596,6 +643,8 @@ class BinaryObject(IgniteDataType):
@classmethod
def from_python(cls, value: object):
+ if value is None:
+ return Null.from_python()
if getattr(value, '_buffer', None) is None:
client = cls.find_client()
diff --git a/pyignite/datatypes/internal.py b/pyignite/datatypes/internal.py
index 9f23ec6..23b9cc4 100644
--- a/pyignite/datatypes/internal.py
+++ b/pyignite/datatypes/internal.py
@@ -479,7 +479,10 @@ class AnyDataArray(AnyDataObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
super().to_python(
getattr(ctype_object, 'element_{}'.format(i)),
diff --git a/pyignite/datatypes/primitive_arrays.py b/pyignite/datatypes/primitive_arrays.py
index 3763b96..1b41728 100644
--- a/pyignite/datatypes/primitive_arrays.py
+++ b/pyignite/datatypes/primitive_arrays.py
@@ -17,6 +17,7 @@ import ctypes
from typing import Any
from pyignite.constants import *
+from . import Null
from .base import IgniteDataType
from .primitive import *
from .type_codes import *
@@ -61,8 +62,13 @@ class PrimitiveArray(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header_class()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
final_class = type(
cls.__name__,
@@ -82,12 +88,18 @@ class PrimitiveArray(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(ctype_object.data[i])
return result
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
@@ -112,7 +124,10 @@ class ByteArray(PrimitiveArray):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return bytearray(ctype_object.data)
+ data = getattr(ctype_object, "data", None)
+ if data is None:
+ return None
+ return bytearray(data)
@classmethod
def from_python(cls, value):
@@ -210,6 +225,9 @@ class ByteArrayObject(PrimitiveArrayObject):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
header_class = cls.build_header_class()
header = header_class()
header.type_code = int.from_bytes(
@@ -282,6 +300,8 @@ class CharArrayObject(PrimitiveArrayObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
values = super().to_python(ctype_object, *args, **kwargs)
+ if values is None:
+ return None
return [
v.to_bytes(
ctypes.sizeof(cls.primitive_type.c_type),
@@ -302,7 +322,10 @@ class BoolArrayObject(PrimitiveArrayObject):
def to_python(cls, ctype_object, *args, **kwargs):
if not ctype_object:
return None
- result = [False] * ctype_object.length
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ result = [False] * length
+ for i in range(length):
result[i] = ctype_object.data[i] != 0
return result
diff --git a/pyignite/datatypes/primitive_objects.py b/pyignite/datatypes/primitive_objects.py
index 033ac9e..53f12d2 100644
--- a/pyignite/datatypes/primitive_objects.py
+++ b/pyignite/datatypes/primitive_objects.py
@@ -17,10 +17,12 @@ import ctypes
from pyignite.constants import *
from pyignite.utils import unsigned
+
from .base import IgniteDataType
from .type_codes import *
from .type_ids import *
from .type_names import *
+from .null_object import Null
__all__ = [
@@ -60,16 +62,21 @@ class DataObject(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
data_type = cls.build_c_type()
- buffer = client.recv(ctypes.sizeof(data_type))
+ buffer = tc_type + client.recv(ctypes.sizeof(data_type) - len(tc_type))
return data_type, buffer
@staticmethod
def to_python(ctype_object, *args, **kwargs):
- return ctype_object.value
+ return getattr(ctype_object, "value", None)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
@@ -185,13 +192,18 @@ class CharObject(DataObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.value.to_bytes(
+ value = getattr(ctype_object, "value", None)
+ if value is None:
+ return None
+ return value.to_bytes(
ctypes.sizeof(cls.c_type),
byteorder=PROTOCOL_BYTE_ORDER
).decode(PROTOCOL_CHAR_ENCODING)
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
if type(value) is str:
value = value.encode(PROTOCOL_CHAR_ENCODING)
# assuming either a bytes or an integer
@@ -218,5 +230,8 @@ class BoolObject(DataObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- return ctype_object.value != 0
+ value = getattr(ctype_object, "value", None)
+ if value is None:
+ return None
+ return value != 0
diff --git a/pyignite/datatypes/standard.py b/pyignite/datatypes/standard.py
index c65cae4..0f16735 100644
--- a/pyignite/datatypes/standard.py
+++ b/pyignite/datatypes/standard.py
@@ -276,8 +276,6 @@ class UUIDObject(StandardObject):
UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
- UUID_BYTE_ORDER = (7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)
-
@staticmethod
def hashcode(value: 'UUID', *args, **kwargs) -> int:
msb = value.int >> 64
@@ -303,6 +301,9 @@ class UUIDObject(StandardObject):
@classmethod
def from_python(cls, value: uuid.UUID):
+ if value is None:
+ return Null.from_python()
+
data_type = cls.build_c_type()
data_object = data_type()
data_object.type_code = int.from_bytes(
@@ -548,8 +549,6 @@ class EnumObject(StandardObject):
cls.type_code,
byteorder=PROTOCOL_BYTE_ORDER
)
- if value is None:
- return Null.from_python(value)
data_object.type_id, data_object.ordinal = value
return bytes(data_object)
@@ -601,8 +600,13 @@ class StandardArray(IgniteDataType):
@classmethod
def parse(cls, client: 'Client'):
+ tc_type = client.recv(ctypes.sizeof(ctypes.c_byte))
+
+ if tc_type == TC_NULL:
+ return Null.build_c_type(), tc_type
+
header_class = cls.build_header_class()
- buffer = client.recv(ctypes.sizeof(header_class))
+ buffer = tc_type + client.recv(ctypes.sizeof(header_class) - len(tc_type))
header = header_class.from_buffer_copy(buffer)
fields = []
for i in range(header.length):
@@ -623,7 +627,10 @@ class StandardArray(IgniteDataType):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
result = []
- for i in range(ctype_object.length):
+ length = getattr(ctype_object, "length", None)
+ if length is None:
+ return None
+ for i in range(length):
result.append(
cls.standard_type.to_python(
getattr(ctype_object, 'element_{}'.format(i)),
@@ -634,6 +641,8 @@ class StandardArray(IgniteDataType):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
header_class = cls.build_header_class()
header = header_class()
if hasattr(header, 'type_code'):
@@ -796,6 +805,9 @@ class EnumArrayObject(StandardArrayObject):
@classmethod
def from_python(cls, value):
+ if value is None:
+ return Null.from_python()
+
type_id, value = value
header_class = cls.build_header_class()
header = header_class()
@@ -815,7 +827,9 @@ class EnumArrayObject(StandardArrayObject):
@classmethod
def to_python(cls, ctype_object, *args, **kwargs):
- type_id = ctype_object.type_id
+ type_id = getattr(ctype_object, "type_id", None)
+ if type_id is None:
+ return None
return type_id, super().to_python(ctype_object, *args, **kwargs)
diff --git a/tests/test_binary.py b/tests/test_binary.py
index 4c45afb..9e798fa 100644
--- a/tests/test_binary.py
+++ b/tests/test_binary.py
@@ -18,7 +18,12 @@ from decimal import Decimal
from pyignite import GenericObjectMeta
from pyignite.datatypes import (
- BinaryObject, BoolObject, IntObject, DecimalObject, LongObject, String,
+ BinaryObject, BoolObject, IntObject, DecimalObject, LongObject, String, ByteObject, ShortObject, FloatObject,
+ DoubleObject, CharObject, UUIDObject, DateObject, TimestampObject, TimeObject, EnumObject, BinaryEnumObject,
+ ByteArrayObject, ShortArrayObject, IntArrayObject, LongArrayObject, FloatArrayObject, DoubleArrayObject,
+ CharArrayObject, BoolArrayObject, UUIDArrayObject, DateArrayObject, TimestampArrayObject, TimeArrayObject,
+ EnumArrayObject, StringArrayObject, DecimalArrayObject, ObjectArrayObject, CollectionObject, MapObject,
+ WrappedDataObject,
)
from pyignite.datatypes.prop_codes import *
@@ -308,8 +313,8 @@ def test_complex_object_names(client):
def test_complex_object_hash(client):
"""
- Test that Python client correctly calculates hash of the binary
- object that contains negative bytes.
+ Test that Python client correctly calculates hash of the binary object that
+ contains negative bytes.
"""
class Internal(
metaclass=GenericObjectMeta,
@@ -355,3 +360,98 @@ def test_complex_object_hash(client):
hash_utf8 = BinaryObject.hashcode(obj_utf8, client=client)
assert hash_utf8 == -1945378474, 'Invalid hashcode value for object with UTF-8 strings'
+
+
+def test_complex_object_null_fields(client):
+ """
+ Test that Python client can correctly write and read binary object that
+ contains null fields.
+ """
+ class AllTypesObject(
+ metaclass=GenericObjectMeta,
+ type_name='TestObject',
+ schema=OrderedDict([
+ ('byteField', ByteObject),
+ ('shortField', ShortObject),
+ ('intField', IntObject),
+ ('longField', LongObject),
+ ('floatField', FloatObject),
+ ('doubleField', DoubleObject),
+ ('charField', CharObject),
+ ('boolField', BoolObject),
+ ('uuidField', UUIDObject),
+ ('dateField', DateObject),
+ ('timestampField', TimestampObject),
+ ('timeField', TimeObject),
+ ('enumField', EnumObject),
+ ('binaryEnumField', BinaryEnumObject),
+ ('byteArrayField', ByteArrayObject),
+ ('shortArrayField', ShortArrayObject),
+ ('intArrayField', IntArrayObject),
+ ('longArrayField', LongArrayObject),
+ ('floatArrayField', FloatArrayObject),
+ ('doubleArrayField', DoubleArrayObject),
+ ('charArrayField', CharArrayObject),
+ ('boolArrayField', BoolArrayObject),
+ ('uuidArrayField', UUIDArrayObject),
+ ('dateArrayField', DateArrayObject),
+ ('timestampArrayField', TimestampArrayObject),
+ ('timeArrayField', TimeArrayObject),
+ ('enumArrayField', EnumArrayObject),
+ ('stringField', String),
+ ('stringArrayField', StringArrayObject),
+ ('decimalField', DecimalObject),
+ ('decimalArrayField', DecimalArrayObject),
+ ('objectArrayField', ObjectArrayObject),
+ ('collectionField', CollectionObject),
+ ('mapField', MapObject),
+ ('binaryObjectField', BinaryObject),
+ ])
+ ):
+ pass
+
+ key = 42
+ null_fields_value = AllTypesObject()
+
+ null_fields_value.byteField = None
+ null_fields_value.shortField = None
+ null_fields_value.intField = 10
+ null_fields_value.longField = None
+ null_fields_value.floatField = None
+ null_fields_value.doubleField = None
+ null_fields_value.charField = None
+ null_fields_value.boolField = None
+ null_fields_value.uuidField = None
+ null_fields_value.dateField = None
+ null_fields_value.timestampField = None
+ null_fields_value.timeField = None
+ null_fields_value.enumField = None
+ null_fields_value.binaryEnumField = None
+ null_fields_value.byteArrayField = None
+ null_fields_value.shortArrayField = None
+ null_fields_value.intArrayField = None
+ null_fields_value.longArrayField = None
+ null_fields_value.floatArrayField = None
+ null_fields_value.doubleArrayField = None
+ null_fields_value.charArrayField = None
+ null_fields_value.boolArrayField = None
+ null_fields_value.uuidArrayField = None
+ null_fields_value.dateArrayField = None
+ null_fields_value.timestampArrayField = None
+ null_fields_value.timeArrayField = None
+ null_fields_value.enumArrayField = None
+ null_fields_value.stringField = None
+ null_fields_value.stringArrayField = None
+ null_fields_value.decimalField = None
+ null_fields_value.decimalArrayField = None
+ null_fields_value.objectArrayField = None
+ null_fields_value.collectionField = None
+ null_fields_value.mapField = None
+ null_fields_value.binaryObjectField = None
+
+ cache = client.get_or_create_cache('all_types_test_cache')
+ cache.put(key, null_fields_value)
+
+ got_obj = cache.get(key)
+
+ assert got_obj == null_fields_value, 'Objects mismatch'