You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by is...@apache.org on 2021/02/18 11:32:45 UTC
[ignite-python-thin-client] branch master updated: IGNITE-14186
Implement C module to speedup hashcode
This is an automated email from the ASF dual-hosted git repository.
isapego pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-python-thin-client.git
The following commit(s) were added to refs/heads/master by this push:
new e5ca3fc IGNITE-14186 Implement C module to speedup hashcode
e5ca3fc is described below
commit e5ca3fceb79a6c2e01c7480e56df162088fbedc0
Author: Ivan Dashchinskiy <iv...@gmail.com>
AuthorDate: Thu Feb 18 14:32:03 2021 +0300
IGNITE-14186 Implement C module to speedup hashcode
This closes #17
---
.gitignore | 4 +
MANIFEST.in | 2 +
README.md | 13 ++++
cext/cutils.c | 193 +++++++++++++++++++++++++++++++++++++++++++++++
pyignite/api/binary.py | 16 +---
pyignite/utils.py | 26 ++++++-
requirements/install.txt | 1 -
scripts/build_wheels.sh | 49 ++++++++++++
scripts/create_distr.sh | 86 +++++++++++++++++++++
scripts/create_sdist.sh | 35 +++++++++
setup.py | 140 ++++++++++++++++++++++------------
tests/conftest.py | 23 +++++-
tests/test_cutils.py | 136 +++++++++++++++++++++++++++++++++
tox.ini | 2 +-
14 files changed, 661 insertions(+), 65 deletions(-)
diff --git a/.gitignore b/.gitignore
index d28510c..699c26d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,12 @@
.idea
+.benchmarks
.vscode
.eggs
.pytest_cache
.tox
+*.so
+build
+distr
tests/config/*.xml
junit*.xml
pyignite.egg-info
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..783a2fe
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,2 @@
+recursive-include requirements *
+include README.md
diff --git a/README.md b/README.md
index 24f7b4e..47bd712 100644
--- a/README.md
+++ b/README.md
@@ -36,6 +36,19 @@ $ pip install -r requirements/<your task>.txt
You may also want to consult the `setuptools` manual about using `setup.py`.
+### *optional C extension*
+There is an optional C extension to speedup some computational intensive tasks. If it's compilation fails
+(missing compiler or CPython headers), `pyignite` will be installed without this module.
+
+- On Linux or MacOS X only C compiler is required (`gcc` or `clang`). It compiles during standard setup process.
+- For building universal `wheels` (binary packages) for Linux, just invoke script `./scripts/create_distr.sh`.
+
+ ***NB!* Docker is required.**
+
+ Ready wheels for `x86` and `x86-64` for different python versions (3.6, 3.7, 3.8 and 3.9) will be
+ located in `./distr` directory.
+
+
### Updating from older version
To upgrade an existing package, use the following command:
diff --git a/cext/cutils.c b/cext/cutils.c
new file mode 100644
index 0000000..0106edc
--- /dev/null
+++ b/cext/cutils.c
@@ -0,0 +1,193 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+#include <Python.h>
+
+#ifdef _MSC_VER
+
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+
+#else
+#include <stdint.h>
+#endif
+
+static int32_t FNV1_OFFSET_BASIS = 0x811c9dc5;
+static int32_t FNV1_PRIME = 0x01000193;
+
+
+PyObject* hashcode(PyObject* self, PyObject *args);
+PyObject* schema_id(PyObject* self, PyObject *args);
+
+PyObject* str_hashcode(PyObject* data);
+int32_t str_hashcode_(PyObject* data, int lower);
+PyObject* b_hashcode(PyObject* data);
+
+static PyMethodDef methods[] = {
+ {"hashcode", (PyCFunction) hashcode, METH_VARARGS, ""},
+ {"schema_id", (PyCFunction) schema_id, METH_VARARGS, ""},
+ {NULL, NULL, 0, NULL} /* Sentinel */
+};
+
+static struct PyModuleDef moduledef = {
+ PyModuleDef_HEAD_INIT,
+ "_cutils",
+ 0, /* m_doc */
+ -1, /* m_size */
+ methods, /* m_methods */
+ NULL, /* m_slots */
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL, /* m_free */
+};
+
+static char* hashcode_input_err = "supported only strings, bytearrays, bytes and memoryview";
+static char* schema_id_input_err = "input argument must be dict or int";
+static char* schema_field_type_err = "schema keys must be strings";
+
+PyMODINIT_FUNC PyInit__cutils(void) {
+ return PyModule_Create(&moduledef);
+}
+
+PyObject* hashcode(PyObject* self, PyObject *args) {
+ PyObject* data;
+
+ if (!PyArg_ParseTuple(args, "O", &data)) {
+ return NULL;
+ }
+
+ if (data == Py_None) {
+ return PyLong_FromLong(0);
+ }
+ else if (PyUnicode_CheckExact(data)) {
+ return str_hashcode(data);
+ }
+ else {
+ return b_hashcode(data);
+ }
+}
+
+PyObject* str_hashcode(PyObject* data) {
+ return PyLong_FromLong(str_hashcode_(data, 0));
+}
+
+int32_t str_hashcode_(PyObject *str, int lower) {
+ int32_t res = 0;
+
+ Py_ssize_t sz = PyUnicode_GET_LENGTH(str);
+ if (!sz) {
+ return res;
+ }
+
+ int kind = PyUnicode_KIND(str);
+ void* buf = PyUnicode_DATA(str);
+
+ Py_ssize_t i;
+ for (i = 0; i < sz; i++) {
+ Py_UCS4 ch = PyUnicode_READ(kind, buf, i);
+
+ if (lower) {
+ ch = Py_UNICODE_TOLOWER(ch);
+ }
+
+ res = 31 * res + ch;
+ }
+
+ return res;
+}
+
+PyObject* b_hashcode(PyObject* data) {
+ int32_t res = 1;
+ Py_ssize_t sz; char* buf;
+
+ if (PyBytes_CheckExact(data)) {
+ sz = PyBytes_GET_SIZE(data);
+ buf = PyBytes_AS_STRING(data);
+ }
+ else if (PyByteArray_CheckExact(data)) {
+ sz = PyByteArray_GET_SIZE(data);
+ buf = PyByteArray_AS_STRING(data);
+ }
+ else if (PyMemoryView_Check(data)) {
+ Py_buffer* pyBuf = PyMemoryView_GET_BUFFER(data);
+ sz = pyBuf->len;
+ buf = (char*)pyBuf->buf;
+ }
+ else {
+ PyErr_SetString(PyExc_ValueError, hashcode_input_err);
+ return NULL;
+ }
+
+ Py_ssize_t i;
+ for (i = 0; i < sz; i++) {
+ res = 31 * res + (signed char)buf[i];
+ }
+
+ return PyLong_FromLong(res);
+}
+
+PyObject* schema_id(PyObject* self, PyObject *args) {
+ PyObject* data;
+
+ if (!PyArg_ParseTuple(args, "O", &data)) {
+ return NULL;
+ }
+
+ if (PyLong_CheckExact(data)) {
+ return PyNumber_Long(data);
+ }
+ else if (data == Py_None) {
+ return PyLong_FromLong(0);
+ }
+ else if (PyDict_Check(data)) {
+ Py_ssize_t sz = PyDict_Size(data);
+
+ if (sz == 0) {
+ return PyLong_FromLong(0);
+ }
+
+ int32_t s_id = FNV1_OFFSET_BASIS;
+
+ PyObject *key, *value;
+ Py_ssize_t pos = 0;
+
+ while (PyDict_Next(data, &pos, &key, &value)) {
+ if (!PyUnicode_CheckExact(key)) {
+ PyErr_SetString(PyExc_ValueError, schema_field_type_err);
+ return NULL;
+ }
+
+ int32_t field_id = str_hashcode_(key, 1);
+ s_id ^= field_id & 0xff;
+ s_id *= FNV1_PRIME;
+ s_id ^= (field_id >> 8) & 0xff;
+ s_id *= FNV1_PRIME;
+ s_id ^= (field_id >> 16) & 0xff;
+ s_id *= FNV1_PRIME;
+ s_id ^= (field_id >> 24) & 0xff;
+ s_id *= FNV1_PRIME;
+ }
+
+ return PyLong_FromLong(s_id);
+ }
+ else {
+ PyErr_SetString(PyExc_ValueError, schema_id_input_err);
+ return NULL;
+ }
+}
diff --git a/pyignite/api/binary.py b/pyignite/api/binary.py
index 0e63c17..87a5232 100644
--- a/pyignite/api/binary.py
+++ b/pyignite/api/binary.py
@@ -22,7 +22,7 @@ from pyignite.datatypes.binary import (
from pyignite.datatypes import String, Int, Bool
from pyignite.queries import Query
from pyignite.queries.op_codes import *
-from pyignite.utils import int_overflow, entity_id
+from pyignite.utils import entity_id, schema_id
from .result import APIResult
from ..stream import BinaryStream, READ_BACKWARD
from ..queries.response import Response
@@ -137,7 +137,7 @@ def put_binary_type(
'is_enum': is_enum,
'schema': [],
}
- schema_id = None
+ s_id = None
if is_enum:
data['enums'] = []
for literal, ordinal in schema.items():
@@ -147,7 +147,7 @@ def put_binary_type(
})
else:
# assemble schema and calculate schema ID in one go
- schema_id = FNV1_OFFSET_BASIS if schema else 0
+ s_id = schema_id(schema)
for field_name, data_type in schema.items():
# TODO: check for allowed data types
field_id = entity_id(field_name)
@@ -159,17 +159,9 @@ def put_binary_type(
),
'field_id': field_id,
})
- schema_id ^= (field_id & 0xff)
- schema_id = int_overflow(schema_id * FNV1_PRIME)
- schema_id ^= ((field_id >> 8) & 0xff)
- schema_id = int_overflow(schema_id * FNV1_PRIME)
- schema_id ^= ((field_id >> 16) & 0xff)
- schema_id = int_overflow(schema_id * FNV1_PRIME)
- schema_id ^= ((field_id >> 24) & 0xff)
- schema_id = int_overflow(schema_id * FNV1_PRIME)
data['schema'].append({
- 'schema_id': schema_id,
+ 'schema_id': s_id,
'schema_fields': [
{'schema_field_id': entity_id(x)} for x in schema
],
diff --git a/pyignite/utils.py b/pyignite/utils.py
index 6c636ae..67f164f 100644
--- a/pyignite/utils.py
+++ b/pyignite/utils.py
@@ -23,6 +23,13 @@ from typing import Any, Optional, Type, Tuple, Union
from pyignite.datatypes.base import IgniteDataType
from .constants import *
+FALLBACK = False
+
+try:
+ from pyignite import _cutils
+except ImportError:
+ FALLBACK = True
+
LONG_MASK = 0xffffffff
DIGITS_PER_INT = 9
@@ -91,6 +98,13 @@ def hashcode(data: Union[str, bytes, bytearray, memoryview]) -> int:
:param data: UTF-8-encoded string identifier of binary buffer or byte array
:return: hash code.
"""
+ if FALLBACK:
+ return __hashcode_fallback(data)
+
+ return _cutils.hashcode(data)
+
+
+def __hashcode_fallback(data: Union[str, bytes, bytearray, memoryview]) -> int:
if isinstance(data, str):
"""
For strings we iterate over code point which are of the int type
@@ -147,13 +161,21 @@ def schema_id(schema: Union[int, dict]) -> int:
:param schema: a dict of field names: field types,
:return: schema ID.
"""
- if type(schema) is int:
+ if FALLBACK:
+ return __schema_id_fallback(schema)
+ return _cutils.schema_id(schema)
+
+
+def __schema_id_fallback(schema: Union[int, dict]) -> int:
+ if isinstance(schema, int):
return schema
+
if schema is None:
return 0
+
s_id = FNV1_OFFSET_BASIS if schema else 0
for field_name in schema.keys():
- field_id = entity_id(field_name)
+ field_id = __hashcode_fallback(field_name.lower())
s_id ^= (field_id & 0xff)
s_id = int_overflow(s_id * FNV1_PRIME)
s_id ^= ((field_id >> 8) & 0xff)
diff --git a/requirements/install.txt b/requirements/install.txt
index 9b87ae8..cecea8f 100644
--- a/requirements/install.txt
+++ b/requirements/install.txt
@@ -1,4 +1,3 @@
# these pip packages are necessary for the pyignite to run
-typing==3.6.6; python_version<'3.5'
attrs==18.1.0
diff --git a/scripts/build_wheels.sh b/scripts/build_wheels.sh
new file mode 100755
index 0000000..cf5f760
--- /dev/null
+++ b/scripts/build_wheels.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -u -x
+
+function repair_wheel {
+ wheel="$1"
+ if ! auditwheel show "$wheel"; then
+ echo "Skipping non-platform wheel $wheel"
+ else
+ auditwheel repair "$wheel" --plat "$PLAT" -w /wheels
+ fi
+}
+
+# Compile wheels
+for PYBIN in /opt/python/*/bin; do
+ if [[ $PYBIN =~ ^(.*)cp3[6789](.*)$ ]]; then
+ "${PYBIN}/pip" wheel /pyignite/ --no-deps -w /wheels
+ fi
+done
+
+# Bundle external shared libraries into the wheels
+for whl in /wheels/*.whl; do
+ repair_wheel "$whl"
+done
+
+for whl in /wheels/*.whl; do
+ if [[ ! $whl =~ ^(.*)manylinux(.*)$ ]]; then
+ rm "$whl"
+ else
+ chmod 666 "$whl"
+ fi
+done
+
+rm -rf /pyignite/*.egg-info
+rm -rf /pyignite/.eggs
diff --git a/scripts/create_distr.sh b/scripts/create_distr.sh
new file mode 100755
index 0000000..5732aba
--- /dev/null
+++ b/scripts/create_distr.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DISTR_DIR="$(pwd)/distr/"
+SRC_DIR="$(pwd)"
+DEFAULT_DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64"
+
+usage() {
+ cat <<EOF
+create_distr.sh: creates wheels and source distr for different python versions and platforms.
+
+Usage: ${0} [options]
+
+The options are as follows:
+-h|--help
+ Display this help message.
+
+-a|--arch
+ Specify architecture, supported variants: i686,x86,x86_64. Build all supported by default.
+
+-d|--dir
+ Specify directory where to store artifacts. Default $(PWD)/../distr
+
+EOF
+ exit 0
+}
+
+normalize_path() {
+ mkdir -p "$DISTR_DIR"
+ cd "$DISTR_DIR" || exit 1
+ DISTR_DIR="$(pwd)"
+ cd "$SRC_DIR" || exit 1
+ SRC_DIR="$(pwd)"
+}
+
+run_wheel_arch() {
+ if [[ $1 =~ ^(i686|x86)$ ]]; then
+ PLAT="manylinux1_i686"
+ PRE_CMD="linux32"
+ DOCKER_IMAGE="quay.io/pypa/manylinux1_i686"
+ elif [[ $1 =~ ^(x86_64)$ ]]; then
+ PLAT="manylinux1_x86_64"
+ PRE_CMD=""
+ DOCKER_IMAGE="$DEFAULT_DOCKER_IMAGE"
+ else
+ echo "unsupported architecture $1, only x86(i686) and x86_64 supported"
+ exit 1
+ fi
+
+ WHEEL_DIR="$DISTR_DIR/$1"
+ mkdir -p "$WHEEL_DIR"
+ docker run --rm -e PLAT=$PLAT -v "$SRC_DIR":/pyignite -v "$WHEEL_DIR":/wheels $DOCKER_IMAGE $PRE_CMD /pyignite/scripts/build_wheels.sh
+}
+
+while [[ $# -ge 1 ]]; do
+ case "$1" in
+ -h|--help) usage;;
+ -a|--arch) ARCH="$2"; shift 2;;
+ -d|--dir) DISTR_DIR="$2"; shift 2;;
+ *) break;;
+ esac
+done
+
+normalize_path
+
+docker run --rm -v "$SRC_DIR":/pyignite -v "$DISTR_DIR":/dist $DEFAULT_DOCKER_IMAGE /pyignite/scripts/create_sdist.sh
+
+if [[ -n "$ARCH" ]]; then
+ run_wheel_arch "$ARCH"
+else
+ run_wheel_arch "x86"
+ run_wheel_arch "x86_64"
+fi
diff --git a/scripts/create_sdist.sh b/scripts/create_sdist.sh
new file mode 100755
index 0000000..d3bd598
--- /dev/null
+++ b/scripts/create_sdist.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -u -x
+
+# Create source dist.
+for PYBIN in /opt/python/*/bin; do
+ if [[ $PYBIN =~ ^(.*)cp3[6789](.*)$ ]]; then
+ cd pyignite
+ "${PYBIN}/python" setup.py sdist --formats=gztar,zip --dist-dir /dist
+ break;
+ fi
+done
+
+for archive in /dist/*; do
+ if [[ $archive =~ ^(.*)(tar\.gz|zip)$ ]]; then
+ chmod 666 "$archive"
+ fi
+done
+
+rm -rf /pyignite/*.egg-info
+rm -rf /pyignite/.eggs
diff --git a/setup.py b/setup.py
index 583eaa3..4d90e4e 100644
--- a/setup.py
+++ b/setup.py
@@ -14,28 +14,45 @@
# limitations under the License.
from collections import defaultdict
+from distutils.command.build_ext import build_ext
+from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError
+
import setuptools
import sys
-PYTHON_REQUIRED = (3, 4)
-PYTHON_INSTALLED = sys.version_info[:2]
+cext = setuptools.Extension(
+ "pyignite._cutils",
+ sources=[
+ "./cext/cutils.c"
+ ],
+ include_dirs=["./cext"]
+)
-if PYTHON_INSTALLED < PYTHON_REQUIRED:
- sys.stderr.write('''
+if sys.platform == 'win32':
+ ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError, ValueError)
+else:
+ ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
-`pyignite` is not compatible with Python {}.{}!
-Use Python {}.{} or above.
+class BuildFailed(Exception):
+ pass
-'''.format(
- PYTHON_INSTALLED[0],
- PYTHON_INSTALLED[1],
- PYTHON_REQUIRED[0],
- PYTHON_REQUIRED[1],
- )
- )
- sys.exit(1)
+
+class ve_build_ext(build_ext):
+ # This class allows C extension building to fail.
+
+ def run(self):
+ try:
+ build_ext.run(self)
+ except DistutilsPlatformError:
+ raise BuildFailed()
+
+ def build_extension(self, ext):
+ try:
+ build_ext.build_extension(self, ext)
+ except ext_errors:
+ raise BuildFailed()
def is_a_requirement(line):
@@ -52,6 +69,7 @@ requirement_sections = [
'tests',
'docs',
]
+
requirements = defaultdict(list)
for section in requirement_sections:
@@ -68,37 +86,63 @@ for section in requirement_sections:
with open('README.md', 'r', encoding='utf-8') as readme_file:
long_description = readme_file.read()
-setuptools.setup(
- name='pyignite',
- version='0.3.4',
- python_requires='>={}.{}'.format(*PYTHON_REQUIRED),
- author='Dmitry Melnichuk',
- author_email='dmitry.melnichuk@nobitlost.com',
- description='Apache Ignite binary client Python API',
- long_description=long_description,
- long_description_content_type='text/markdown',
- url=(
- 'https://github.com/apache/ignite/tree/master'
- '/modules/platforms/python'
- ),
- packages=setuptools.find_packages(),
- install_requires=requirements['install'],
- tests_require=requirements['tests'],
- setup_requires=requirements['setup'],
- extras_require={
- 'docs': requirements['docs'],
- },
- classifiers=[
- 'Programming Language :: Python',
- 'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.4',
- 'Programming Language :: Python :: 3.5',
- 'Programming Language :: Python :: 3.6',
- 'Programming Language :: Python :: 3 :: Only',
- 'Intended Audience :: Developers',
- 'Topic :: Database :: Front-Ends',
- 'Topic :: Software Development :: Libraries :: Python Modules',
- 'License :: OSI Approved :: Apache Software License',
- 'Operating System :: OS Independent',
- ],
-)
+
+def run_setup(with_binary=True):
+ if with_binary:
+ kw = dict(
+ ext_modules=[cext],
+ cmdclass=dict(build_ext=ve_build_ext),
+ )
+ else:
+ kw = dict()
+
+ setuptools.setup(
+ name='pyignite',
+ version='0.4.0',
+ python_requires='>=3.6',
+ author='The Apache Software Foundation',
+ author_email='dev@ignite.apache.org',
+ description='Apache Ignite binary client Python API',
+ url='https://github.com/apache/ignite-python-thin-client',
+ packages=setuptools.find_packages(),
+ install_requires=requirements['install'],
+ tests_require=requirements['tests'],
+ setup_requires=requirements['setup'],
+ extras_require={
+ 'docs': requirements['docs'],
+ },
+ classifiers=[
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
+ 'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: 3.9',
+ 'Programming Language :: Python :: 3 :: Only',
+ 'Intended Audience :: Developers',
+ 'Topic :: Database :: Front-Ends',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ ],
+ **kw
+ )
+
+
+try:
+ run_setup()
+except BuildFailed:
+ BUILD_EXT_WARNING = ("WARNING: The C extension could not be compiled, "
+ "speedups are not enabled.")
+ print('*' * 75)
+ print(BUILD_EXT_WARNING)
+ print("Failure information, if any, is above.")
+ print("I'm retrying the build without the C extension now.")
+ print('*' * 75)
+
+ run_setup(False)
+
+ print('*' * 75)
+ print(BUILD_EXT_WARNING)
+ print("Plain python installation succeeded.")
+ print('*' * 75)
diff --git a/tests/conftest.py b/tests/conftest.py
index bc8804d..bd86f9c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -197,6 +197,21 @@ def run_examples(request, examples):
pytest.skip('skipped examples: --examples is not passed')
+@pytest.fixture(autouse=True)
+def skip_if_no_cext(request):
+ skip = False
+ try:
+ from pyignite import _cutils
+ except ImportError:
+ if request.config.getoption('--force-cext'):
+ pytest.fail("C extension failed to build, fail test because of --force-cext is set.")
+ return
+ skip = True
+
+ if skip and request.node.get_closest_marker('skip_if_no_cext'):
+ pytest.skip('skipped c extensions test, c extension is not available.')
+
+
def pytest_addoption(parser):
parser.addoption(
'--node',
@@ -300,6 +315,11 @@ def pytest_addoption(parser):
action='store_true',
help='check if examples can be run',
)
+ parser.addoption(
+ '--force-cext',
+ action='store_true',
+ help='check if examples can be run',
+ )
def pytest_generate_tests(metafunc):
@@ -334,5 +354,6 @@ def pytest_generate_tests(metafunc):
def pytest_configure(config):
config.addinivalue_line(
- "markers", "examples: mark test to run only if --examples are set"
+ "markers", "examples: mark test to run only if --examples are set\n"
+ "skip_if_no_cext: mark test to run only if c extension is available"
)
diff --git a/tests/test_cutils.py b/tests/test_cutils.py
new file mode 100644
index 0000000..e7c095e
--- /dev/null
+++ b/tests/test_cutils.py
@@ -0,0 +1,136 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+from collections import OrderedDict
+
+import pytest
+
+import pyignite.utils as _putils
+from pyignite.datatypes import IntObject
+
+try:
+ from pyignite import _cutils
+
+ _cutils_hashcode = _cutils.hashcode
+ _cutils_schema_id = _cutils.schema_id
+except ImportError:
+ _cutils_hashcode = lambda x: None
+ _cutils_schema_id = lambda x: None
+ pass
+
+
+@pytest.mark.skip_if_no_cext
+def test_bytes_hashcode():
+ assert _cutils_hashcode(None) == 0
+ assert _cutils_hashcode(b'') == 1
+ assert _cutils_hashcode(bytearray()) == 1
+ assert _cutils_hashcode(memoryview(b'')) == 1
+
+ for i in range(1000):
+ rnd_bytes = bytearray([random.randint(0, 255) for _ in range(random.randint(1, 1024))])
+
+ fallback_val = _putils.__hashcode_fallback(rnd_bytes)
+ assert _cutils_hashcode(rnd_bytes) == fallback_val
+ assert _cutils_hashcode(bytes(rnd_bytes)) == fallback_val
+ assert _cutils_hashcode(memoryview(rnd_bytes)) == fallback_val
+
+
+@pytest.mark.skip_if_no_cext
+@pytest.mark.parametrize(
+ 'value',
+ [
+ '皮膚の色、',
+ 'Произвольный символ',
+ 'Random string',
+ '',
+ ]
+)
+def test_string_hashcode(value):
+ assert _cutils_hashcode(value) == _putils.__hashcode_fallback(value), f'failed on {value}'
+
+
+@pytest.mark.skip_if_no_cext
+def test_random_string_hashcode():
+ assert _cutils_hashcode(None) == 0
+ assert _cutils_hashcode('') == 0
+
+ for i in range(1000):
+ rnd_str = get_random_unicode(random.randint(1, 128))
+ assert _cutils_hashcode(rnd_str) == _putils.__hashcode_fallback(rnd_str), f'failed on {rnd_str}'
+
+
+@pytest.mark.skip_if_no_cext
+def test_schema_id():
+ rnd_id = random.randint(-100, 100)
+ assert _cutils_schema_id(rnd_id) == rnd_id
+ assert _cutils_schema_id(None) == 0
+ assert _cutils_schema_id({}) == 0
+
+ for i in range(1000):
+ schema = OrderedDict({get_random_field_name(20): IntObject for _ in range(20)})
+ assert _cutils_schema_id(schema) == _putils.__schema_id_fallback(schema), f'failed on {schema}'
+
+
+@pytest.mark.skip_if_no_cext
+@pytest.mark.parametrize(
+ 'func,args,kwargs,err_cls',
+ [
+ [_cutils_hashcode, [123], {}, ValueError],
+ [_cutils_hashcode, [{'test': 'test'}], {}, ValueError],
+ [_cutils_hashcode, [], {}, TypeError],
+ [_cutils_hashcode, [123, 123], {}, TypeError],
+ [_cutils_hashcode, [], {'input': 'test'}, TypeError],
+ [_cutils_schema_id, ['test'], {}, ValueError],
+ [_cutils_schema_id, [], {}, TypeError],
+ [_cutils_schema_id, [], {}, TypeError],
+ [_cutils_schema_id, [123, 123], {}, TypeError],
+ [_cutils_schema_id, [], {'input': 'test'}, TypeError],
+ ]
+)
+def test_handling_errors(func, args, kwargs, err_cls):
+ with pytest.raises(err_cls):
+ func(*args, **kwargs)
+
+
+def get_random_field_name(length):
+ first = get_random_unicode(length // 2, latin=True)
+ second = get_random_unicode(length - length // 2, latin=True)
+
+ first = first.upper() if random.randint(0, 1) else first.lower()
+ second = second.upper() if random.randint(0, 1) else second.lower()
+
+ return first + '_' + second
+
+
+def get_random_unicode(length, latin=False):
+ include_ranges = [
+ (0x0041, 0x005A), # Latin high
+ (0x0061, 0x007A), # Latin lower
+ (0x0410, 0x042F), # Russian high
+ (0x0430, 0x044F), # Russian lower
+ (0x05D0, 0x05EA) # Hebrew
+ ]
+
+ alphabet = []
+
+ if latin:
+ include_ranges = include_ranges[0:2]
+
+ for current_range in include_ranges:
+ for code_point in range(current_range[0], current_range[1] + 1):
+ alphabet.append(chr(code_point))
+
+ return ''.join(random.choice(alphabet) for _ in range(length))
diff --git a/tox.ini b/tox.ini
index eb7d1a6..104a705 100644
--- a/tox.ini
+++ b/tox.ini
@@ -26,7 +26,7 @@ deps =
recreate = True
usedevelop = True
commands =
- pytest {env:PYTESTARGS:} {posargs}
+ pytest {env:PYTESTARGS:} {posargs} --force-cext
[jenkins]
setenv: