You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by ni...@apache.org on 2020/07/20 19:26:17 UTC
[incubator-heron] branch master updated: Unvendor cloudpickle
(#3568)
This is an automated email from the ASF dual-hosted git repository.
nicknezis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 1c7fd11 Unvendor cloudpickle (#3568)
1c7fd11 is described below
commit 1c7fd118e37f3ac97e8664aef8fcb0c65bfcf5df
Author: Oliver Bristow <ev...@gmail.com>
AuthorDate: Mon Jul 20 20:26:06 2020 +0100
Unvendor cloudpickle (#3568)
---
LICENSE | 2 -
heronpy/api/BUILD | 3 +
heronpy/api/cloudpickle.py | 1345 ----------------------
heronpy/api/serializer.py | 2 +-
licenses/LICENSE-cloudpickle.txt | 32 -
scripts/packages/heronpy/__apiinit__.py.template | 1 -
scripts/packages/heronpy/requirements.txt | 1 +
7 files changed, 5 insertions(+), 1381 deletions(-)
diff --git a/LICENSE b/LICENSE
index 5d86258..c9e6076 100644
--- a/LICENSE
+++ b/LICENSE
@@ -311,8 +311,6 @@ See project link for details.
-> heron/tools/ui/resources/static/js/JSXTransformer.0.10.0.js
autogen.sh
-> config/autogen.sh
- cloudpickle(https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE)
- -> heronpy/api/cloudpickle.py
cpplint(https://github.com/cpplint/cpplint/blob/master/LICENSE)
-> third_party/python/cpplint/cpplint.py
d3(v3.4.11, https://github.com/d3/d3/blob/master/LICENSE)
diff --git a/heronpy/api/BUILD b/heronpy/api/BUILD
index 2e6001a..2b55b75 100644
--- a/heronpy/api/BUILD
+++ b/heronpy/api/BUILD
@@ -9,6 +9,9 @@ pex_library(
deps = [
"//heronpy/proto:proto-py",
],
+ reqs = [
+ "cloudpickle~=1.5.0",
+ ],
)
# for egg production
diff --git a/heronpy/api/cloudpickle.py b/heronpy/api/cloudpickle.py
deleted file mode 100644
index 27a2180..0000000
--- a/heronpy/api/cloudpickle.py
+++ /dev/null
@@ -1,1345 +0,0 @@
-"""
-This class is defined to override standard pickle functionality
-The goals of it follow:
--Serialize lambdas and nested functions to compiled byte code
--Deal with main module correctly
--Deal with other non-serializable objects
-It does not include an unpickler, as standard python unpickling suffices.
-This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
-<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
-Copyright (c) 2012, Regents of the University of California.
-Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- * Neither the name of the University of California, Berkeley nor the
- names of its contributors may be used to endorse or promote
- products derived from this software without specific prior written
- permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
-TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-"""
-# pylint: skip-file
-from __future__ import print_function
-
-import abc
-import builtins
-import dis
-import io
-import itertools
-import logging
-import opcode
-import operator
-import pickle
-import platform
-import struct
-import sys
-import types
-import weakref
-import uuid
-import threading
-import typing
-from enum import Enum
-
-from typing import Generic, Union, Tuple, Callable
-from pickle import _Pickler as Pickler
-from pickle import _getattribute
-from io import BytesIO
-from importlib._bootstrap import _find_spec
-
-try: # pragma: no branch
- import typing_extensions as _typing_extensions
- from typing_extensions import Literal, Final
-except ImportError:
- _typing_extensions = Literal = Final = None
-
-if sys.version_info >= (3, 5, 3):
- from typing import ClassVar
-else: # pragma: no cover
- ClassVar = None
-
-
-# cloudpickle is meant for inter process communication: we expect all
-# communicating processes to run the same Python version hence we favor
-# communication speed over compatibility:
-DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
-
-# Track the provenance of reconstructed dynamic classes to make it possible to
-# recontruct instances from the matching singleton class definition when
-# appropriate and preserve the usual "isinstance" semantics of Python objects.
-_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
-_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
-_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
-
-PYPY = platform.python_implementation() == "PyPy"
-
-builtin_code_type = None
-if PYPY:
- # builtin-code objects only exist in pypy
- builtin_code_type = type(float.__new__.__code__)
-
-_extract_code_globals_cache = weakref.WeakKeyDictionary()
-
-
-def _get_or_create_tracker_id(class_def):
- with _DYNAMIC_CLASS_TRACKER_LOCK:
- class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
- if class_tracker_id is None:
- class_tracker_id = uuid.uuid4().hex
- _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
- _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
- return class_tracker_id
-
-
-def _lookup_class_or_track(class_tracker_id, class_def):
- if class_tracker_id is not None:
- with _DYNAMIC_CLASS_TRACKER_LOCK:
- class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
- class_tracker_id, class_def)
- _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
- return class_def
-
-
-def _whichmodule(obj, name):
- """Find the module an object belongs to.
- This function differs from ``pickle.whichmodule`` in two ways:
- - it does not mangle the cases where obj's module is __main__ and obj was
- not found in any module.
- - Errors arising during module introspection are ignored, as those errors
- are considered unwanted side effects.
- """
- if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa
- # Workaround bug in old Python versions: prior to Python 3.7,
- # T.__module__ would always be set to "typing" even when the TypeVar T
- # would be defined in a different module.
- #
- # For such older Python versions, we ignore the __module__ attribute of
- # TypeVar instances and instead exhaustively lookup those instances in
- # all currently imported modules.
- module_name = None
- else:
- module_name = getattr(obj, '__module__', None)
-
- if module_name is not None:
- return module_name
- # Protect the iteration by using a copy of sys.modules against dynamic
- # modules that trigger imports of other modules upon calls to getattr or
- # other threads importing at the same time.
- for module_name, module in sys.modules.copy().items():
- # Some modules such as coverage can inject non-module objects inside
- # sys.modules
- if (
- module_name == '__main__' or
- module is None or
- not isinstance(module, types.ModuleType)
- ):
- continue
- try:
- if _getattribute(module, name)[0] is obj:
- return module_name
- except Exception:
- pass
- return None
-
-
-def _is_importable_by_name(obj, name=None):
- """Determine if obj can be pickled as attribute of a file-backed module"""
- return _lookup_module_and_qualname(obj, name=name) is not None
-
-
-def _lookup_module_and_qualname(obj, name=None):
- if name is None:
- name = getattr(obj, '__qualname__', None)
- if name is None: # pragma: no cover
- # This used to be needed for Python 2.7 support but is probably not
- # needed anymore. However we keep the __name__ introspection in case
- # users of cloudpickle rely on this old behavior for unknown reasons.
- name = getattr(obj, '__name__', None)
-
- module_name = _whichmodule(obj, name)
-
- if module_name is None:
- # In this case, obj.__module__ is None AND obj was not found in any
- # imported module. obj is thus treated as dynamic.
- return None
-
- if module_name == "__main__":
- return None
-
- module = sys.modules.get(module_name, None)
- if module is None:
- # The main reason why obj's module would not be imported is that this
- # module has been dynamically created, using for example
- # types.ModuleType. The other possibility is that module was removed
- # from sys.modules after obj was created/imported. But this case is not
- # supported, as the standard pickle does not support it either.
- return None
-
- # module has been added to sys.modules, but it can still be dynamic.
- if _is_dynamic(module):
- return None
-
- try:
- obj2, parent = _getattribute(module, name)
- except AttributeError:
- # obj was not found inside the module it points to
- return None
- if obj2 is not obj:
- return None
- return module, name
-
-
-def _extract_code_globals(co):
- """
- Find all globals names read or written to by codeblock co
- """
- out_names = _extract_code_globals_cache.get(co)
- if out_names is None:
- names = co.co_names
- out_names = {names[oparg] for _, oparg in _walk_global_ops(co)}
-
- # Declaring a function inside another one using the "def ..."
- # syntax generates a constant code object corresonding to the one
- # of the nested function's As the nested function may itself need
- # global variables, we need to introspect its code, extract its
- # globals, (look for code object in it's co_consts attribute..) and
- # add the result to code_globals
- if co.co_consts:
- for const in co.co_consts:
- if isinstance(const, types.CodeType):
- out_names |= _extract_code_globals(const)
-
- _extract_code_globals_cache[co] = out_names
-
- return out_names
-
-
-def _find_imported_submodules(code, top_level_dependencies):
- """
- Find currently imported submodules used by a function.
- Submodules used by a function need to be detected and referenced for the
- function to work correctly at depickling time. Because submodules can be
- referenced as attribute of their parent package (``package.submodule``), we
- need a special introspection technique that does not rely on GLOBAL-related
- opcodes to find references of them in a code object.
- Example:
- ```
- import concurrent.futures
- import cloudpickle
- def func():
- x = concurrent.futures.ThreadPoolExecutor
- if __name__ == '__main__':
- cloudpickle.dumps(func)
- ```
- The globals extracted by cloudpickle in the function's state include the
- concurrent package, but not its submodule (here, concurrent.futures), which
- is the module used by func. Find_imported_submodules will detect the usage
- of concurrent.futures. Saving this module alongside with func will ensure
- that calling func once depickled does not fail due to concurrent.futures
- not being imported
- """
-
- subimports = []
- # check if any known dependency is an imported package
- for x in top_level_dependencies:
- if (isinstance(x, types.ModuleType) and
- hasattr(x, '__package__') and x.__package__):
- # check if the package has any currently loaded sub-imports
- prefix = x.__name__ + '.'
- # A concurrent thread could mutate sys.modules,
- # make sure we iterate over a copy to avoid exceptions
- for name in list(sys.modules):
- # Older versions of pytest will add a "None" module to
- # sys.modules.
- if name is not None and name.startswith(prefix):
- # check whether the function can address the sub-module
- tokens = set(name[len(prefix):].split('.'))
- if not tokens - set(code.co_names):
- subimports.append(sys.modules[name])
- return subimports
-
-
-def cell_set(cell, value):
- """Set the value of a closure cell.
- The point of this function is to set the cell_contents attribute of a cell
- after its creation. This operation is necessary in case the cell contains a
- reference to the function the cell belongs to, as when calling the
- function's constructor
- ``f = types.FunctionType(code, globals, name, argdefs, closure)``,
- closure will not be able to contain the yet-to-be-created f.
- In Python3.7, cell_contents is writeable, so setting the contents of a cell
- can be done simply using
- >>> cell.cell_contents = value
- In earlier Python3 versions, the cell_contents attribute of a cell is read
- only, but this limitation can be worked around by leveraging the Python 3
- ``nonlocal`` keyword.
- In Python2 however, this attribute is read only, and there is no
- ``nonlocal`` keyword. For this reason, we need to come up with more
- complicated hacks to set this attribute.
- The chosen approach is to create a function with a STORE_DEREF opcode,
- which sets the content of a closure variable. Typically:
- >>> def inner(value):
- ... lambda: cell # the lambda makes cell a closure
- ... cell = value # cell is a closure, so this triggers a STORE_DEREF
- (Note that in Python2, A STORE_DEREF can never be triggered from an inner
- function. The function g for example here
- >>> def f(var):
- ... def g():
- ... var += 1
- ... return g
- will not modify the closure variable ``var```inplace, but instead try to
- load a local variable var and increment it. As g does not assign the local
- variable ``var`` any initial value, calling f(1)() will fail at runtime.)
- Our objective is to set the value of a given cell ``cell``. So we need to
- somewhat reference our ``cell`` object into the ``inner`` function so that
- this object (and not the smoke cell of the lambda function) gets affected
- by the STORE_DEREF operation.
- In inner, ``cell`` is referenced as a cell variable (an enclosing variable
- that is referenced by the inner function). If we create a new function
- cell_set with the exact same code as ``inner``, but with ``cell`` marked as
- a free variable instead, the STORE_DEREF will be applied on its closure -
- ``cell``, which we can specify explicitly during construction! The new
- cell_set variable thus actually sets the contents of a specified cell!
- Note: we do not make use of the ``nonlocal`` keyword to set the contents of
- a cell in early python3 versions to limit possible syntax errors in case
- test and checker libraries decide to parse the whole file.
- """
-
- if sys.version_info[:2] >= (3, 7): # pragma: no branch
- cell.cell_contents = value
- else:
- _cell_set = types.FunctionType(
- _cell_set_template_code, {}, '_cell_set', (), (cell,),)
- _cell_set(value)
-
-
-def _make_cell_set_template_code():
- def _cell_set_factory(value):
- lambda: cell
- cell = value
-
- co = _cell_set_factory.__code__
-
- _cell_set_template_code = types.CodeType(
- co.co_argcount,
- co.co_kwonlyargcount, # Python 3 only argument
- co.co_nlocals,
- co.co_stacksize,
- co.co_flags,
- co.co_code,
- co.co_consts,
- co.co_names,
- co.co_varnames,
- co.co_filename,
- co.co_name,
- co.co_firstlineno,
- co.co_lnotab,
- co.co_cellvars, # co_freevars is initialized with co_cellvars
- (), # co_cellvars is made empty
- )
- return _cell_set_template_code
-
-
-if sys.version_info[:2] < (3, 7):
- _cell_set_template_code = _make_cell_set_template_code()
-
-# relevant opcodes
-STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
-DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
-LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
-GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
-HAVE_ARGUMENT = dis.HAVE_ARGUMENT
-EXTENDED_ARG = dis.EXTENDED_ARG
-
-
-_BUILTIN_TYPE_NAMES = {}
-for k, v in types.__dict__.items():
- if type(v) is type:
- _BUILTIN_TYPE_NAMES[v] = k
-
-
-def _builtin_type(name):
- if name == "ClassType": # pragma: no cover
- # Backward compat to load pickle files generated with cloudpickle
- # < 1.3 even if loading pickle files from older versions is not
- # officially supported.
- return type
- return getattr(types, name)
-
-
-def _walk_global_ops(code):
- """
- Yield (opcode, argument number) tuples for all
- global-referencing instructions in *code*.
- """
- for instr in dis.get_instructions(code):
- op = instr.opcode
- if op in GLOBAL_OPS:
- yield op, instr.arg
-
-
-def _extract_class_dict(cls):
- """Retrieve a copy of the dict of a class without the inherited methods"""
- clsdict = dict(cls.__dict__) # copy dict proxy to a dict
- if len(cls.__bases__) == 1:
- inherited_dict = cls.__bases__[0].__dict__
- else:
- inherited_dict = {}
- for base in reversed(cls.__bases__):
- inherited_dict.update(base.__dict__)
- to_remove = []
- for name, value in clsdict.items():
- try:
- base_value = inherited_dict[name]
- if value is base_value:
- to_remove.append(name)
- except KeyError:
- pass
- for name in to_remove:
- clsdict.pop(name)
- return clsdict
-
-
-if sys.version_info[:2] < (3, 7): # pragma: no branch
- def _is_parametrized_type_hint(obj):
- # This is very cheap but might generate false positives.
- # general typing Constructs
- is_typing = getattr(obj, '__origin__', None) is not None
-
- # typing_extensions.Literal
- is_litteral = getattr(obj, '__values__', None) is not None
-
- # typing_extensions.Final
- is_final = getattr(obj, '__type__', None) is not None
-
- # typing.Union/Tuple for old Python 3.5
- is_union = getattr(obj, '__union_params__', None) is not None
- is_tuple = getattr(obj, '__tuple_params__', None) is not None
- is_callable = (
- getattr(obj, '__result__', None) is not None and
- getattr(obj, '__args__', None) is not None
- )
- return any((is_typing, is_litteral, is_final, is_union, is_tuple,
- is_callable))
-
- def _create_parametrized_type_hint(origin, args):
- return origin[args]
-
-
-class CloudPickler(Pickler):
-
- dispatch = Pickler.dispatch.copy()
-
- def __init__(self, file, protocol=None):
- if protocol is None:
- protocol = DEFAULT_PROTOCOL
- Pickler.__init__(self, file, protocol=protocol)
- # map ids to dictionary. used to ensure that functions can share global env
- self.globals_ref = {}
-
- def dump(self, obj):
- self.inject_addons()
- try:
- return Pickler.dump(self, obj)
- except RuntimeError as e:
- if 'recursion' in e.args[0]:
- msg = """Could not pickle object as excessively deep recursion required."""
- raise pickle.PicklingError(msg)
- else:
- raise
-
- def save_typevar(self, obj):
- self.save_reduce(*_typevar_reduce(obj), obj=obj)
-
- dispatch[typing.TypeVar] = save_typevar
-
- def save_memoryview(self, obj):
- self.save(obj.tobytes())
-
- dispatch[memoryview] = save_memoryview
-
- def save_module(self, obj):
- """
- Save a module as an import
- """
- if _is_dynamic(obj):
- obj.__dict__.pop('__builtins__', None)
- self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
- obj=obj)
- else:
- self.save_reduce(subimport, (obj.__name__,), obj=obj)
-
- dispatch[types.ModuleType] = save_module
-
- def save_codeobject(self, obj):
- """
- Save a code object
- """
- if hasattr(obj, "co_posonlyargcount"): # pragma: no branch
- args = (
- obj.co_argcount, obj.co_posonlyargcount,
- obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
- obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
- obj.co_varnames, obj.co_filename, obj.co_name,
- obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
- obj.co_cellvars
- )
- else:
- args = (
- obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
- obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
- obj.co_names, obj.co_varnames, obj.co_filename,
- obj.co_name, obj.co_firstlineno, obj.co_lnotab,
- obj.co_freevars, obj.co_cellvars
- )
- self.save_reduce(types.CodeType, args, obj=obj)
-
- dispatch[types.CodeType] = save_codeobject
-
- def save_function(self, obj, name=None):
- """ Registered with the dispatch to handle all function types.
- Determines what kind of function obj is (e.g. lambda, defined at
- interactive prompt, etc) and handles the pickling appropriately.
- """
- if _is_importable_by_name(obj, name=name):
- return Pickler.save_global(self, obj, name=name)
- elif PYPY and isinstance(obj.__code__, builtin_code_type):
- return self.save_pypy_builtin_func(obj)
- else:
- return self.save_function_tuple(obj)
-
- dispatch[types.FunctionType] = save_function
-
- def save_pypy_builtin_func(self, obj):
- """Save pypy equivalent of builtin functions.
- PyPy does not have the concept of builtin-functions. Instead,
- builtin-functions are simple function instances, but with a
- builtin-code attribute.
- Most of the time, builtin functions should be pickled by attribute. But
- PyPy has flaky support for __qualname__, so some builtin functions such
- as float.__new__ will be classified as dynamic. For this reason only,
- we created this special routine. Because builtin-functions are not
- expected to have closure or globals, there is no additional hack
- (compared the one already implemented in pickle) to protect ourselves
- from reference cycles. A simple (reconstructor, newargs, obj.__dict__)
- tuple is save_reduced.
- Note also that PyPy improved their support for __qualname__ in v3.6, so
- this routing should be removed when cloudpickle supports only PyPy 3.6
- and later.
- """
- rv = (types.FunctionType, (obj.__code__, {}, obj.__name__,
- obj.__defaults__, obj.__closure__),
- obj.__dict__)
- self.save_reduce(*rv, obj=obj)
-
- def _save_dynamic_enum(self, obj, clsdict):
- """Special handling for dynamic Enum subclasses
- Use a dedicated Enum constructor (inspired by EnumMeta.__call__) as the
- EnumMeta metaclass has complex initialization that makes the Enum
- subclasses hold references to their own instances.
- """
- members = dict((e.name, e.value) for e in obj)
-
- self.save_reduce(
- _make_skeleton_enum,
- (obj.__bases__, obj.__name__, obj.__qualname__,
- members, obj.__module__, _get_or_create_tracker_id(obj), None),
- obj=obj
- )
-
- # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
- # Those attributes are already handled by the metaclass.
- for attrname in ["_generate_next_value_", "_member_names_",
- "_member_map_", "_member_type_",
- "_value2member_map_"]:
- clsdict.pop(attrname, None)
- for member in members:
- clsdict.pop(member)
-
- def save_dynamic_class(self, obj):
- """Save a class that can't be stored as module global.
- This method is used to serialize classes that are defined inside
- functions, or that otherwise can't be serialized as attribute lookups
- from global modules.
- """
- clsdict = _extract_class_dict(obj)
- clsdict.pop('__weakref__', None)
-
- if issubclass(type(obj), abc.ABCMeta):
- # If obj is an instance of an ABCMeta subclass, dont pickle the
- # cache/negative caches populated during isinstance/issubclass
- # checks, but pickle the list of registered subclasses of obj.
- clsdict.pop('_abc_cache', None)
- clsdict.pop('_abc_negative_cache', None)
- clsdict.pop('_abc_negative_cache_version', None)
- registry = clsdict.pop('_abc_registry', None)
- if registry is None:
- # in Python3.7+, the abc caches and registered subclasses of a
- # class are bundled into the single _abc_impl attribute
- clsdict.pop('_abc_impl', None)
- (registry, _, _, _) = abc._get_dump(obj)
-
- clsdict["_abc_impl"] = [subclass_weakref()
- for subclass_weakref in registry]
- else:
- # In the above if clause, registry is a set of weakrefs -- in
- # this case, registry is a WeakSet
- clsdict["_abc_impl"] = [type_ for type_ in registry]
-
- # On PyPy, __doc__ is a readonly attribute, so we need to include it in
- # the initial skeleton class. This is safe because we know that the
- # doc can't participate in a cycle with the original class.
- type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}
-
- if "__slots__" in clsdict:
- type_kwargs['__slots__'] = obj.__slots__
- # pickle string length optimization: member descriptors of obj are
- # created automatically from obj's __slots__ attribute, no need to
- # save them in obj's state
- if isinstance(obj.__slots__, str):
- clsdict.pop(obj.__slots__)
- else:
- for k in obj.__slots__:
- clsdict.pop(k, None)
-
- # If type overrides __dict__ as a property, include it in the type
- # kwargs. In Python 2, we can't set this attribute after construction.
- # XXX: can this ever happen in Python 3? If so add a test.
- __dict__ = clsdict.pop('__dict__', None)
- if isinstance(__dict__, property):
- type_kwargs['__dict__'] = __dict__
-
- save = self.save
- write = self.write
-
- # We write pickle instructions explicitly here to handle the
- # possibility that the type object participates in a cycle with its own
- # __dict__. We first write an empty "skeleton" version of the class and
- # memoize it before writing the class' __dict__ itself. We then write
- # instructions to "rehydrate" the skeleton class by restoring the
- # attributes from the __dict__.
- #
- # A type can appear in a cycle with its __dict__ if an instance of the
- # type appears in the type's __dict__ (which happens for the stdlib
- # Enum class), or if the type defines methods that close over the name
- # of the type, (which is common for Python 2-style super() calls).
-
- # Push the rehydration function.
- save(_rehydrate_skeleton_class)
-
- # Mark the start of the args tuple for the rehydration function.
- write(pickle.MARK)
-
- # Create and memoize an skeleton class with obj's name and bases.
- if Enum is not None and issubclass(obj, Enum):
- # Special handling of Enum subclasses
- self._save_dynamic_enum(obj, clsdict)
- else:
- # "Regular" class definition:
- tp = type(obj)
- self.save_reduce(_make_skeleton_class,
- (tp, obj.__name__, _get_bases(obj), type_kwargs,
- _get_or_create_tracker_id(obj), None),
- obj=obj)
-
- # Now save the rest of obj's __dict__. Any references to obj
- # encountered while saving will point to the skeleton class.
- save(clsdict)
-
- # Write a tuple of (skeleton_class, clsdict).
- write(pickle.TUPLE)
-
- # Call _rehydrate_skeleton_class(skeleton_class, clsdict)
- write(pickle.REDUCE)
-
- def save_function_tuple(self, func):
- """ Pickles an actual func object.
- A func comprises: code, globals, defaults, closure, and dict. We
- extract and save these, injecting reducing functions at certain points
- to recreate the func object. Keep in mind that some of these pieces
- can contain a ref to the func itself. Thus, a naive save on these
- pieces could trigger an infinite loop of save's. To get around that,
- we first create a skeleton func object using just the code (this is
- safe, since this won't contain a ref to the func), and memoize it as
- soon as it's created. The other stuff can then be filled in later.
- """
- if is_tornado_coroutine(func):
- self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,),
- obj=func)
- return
-
- save = self.save
- write = self.write
-
- code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func)
-
- save(_fill_function) # skeleton function updater
- write(pickle.MARK) # beginning of tuple that _fill_function expects
-
- # Extract currently-imported submodules used by func. Storing these
- # modules in a smoke _cloudpickle_subimports attribute of the object's
- # state will trigger the side effect of importing these modules at
- # unpickling time (which is necessary for func to work correctly once
- # depickled)
- submodules = _find_imported_submodules(
- code,
- itertools.chain(f_globals.values(), closure_values or ()),
- )
-
- # create a skeleton function object and memoize it
- save(_make_skel_func)
- save((
- code,
- len(closure_values) if closure_values is not None else -1,
- base_globals,
- ))
- write(pickle.REDUCE)
- self.memoize(func)
-
- # save the rest of the func data needed by _fill_function
- state = {
- 'globals': f_globals,
- 'defaults': defaults,
- 'dict': dct,
- 'closure_values': closure_values,
- 'module': func.__module__,
- 'name': func.__name__,
- 'doc': func.__doc__,
- '_cloudpickle_submodules': submodules
- }
- if hasattr(func, '__annotations__'):
- state['annotations'] = func.__annotations__
- if hasattr(func, '__qualname__'):
- state['qualname'] = func.__qualname__
- if hasattr(func, '__kwdefaults__'):
- state['kwdefaults'] = func.__kwdefaults__
- save(state)
- write(pickle.TUPLE)
- write(pickle.REDUCE) # applies _fill_function on the tuple
-
- def extract_func_data(self, func):
- """
- Turn the function into a tuple of data necessary to recreate it:
- code, globals, defaults, closure_values, dict
- """
- code = func.__code__
-
- # extract all global ref's
- func_global_refs = _extract_code_globals(code)
-
- # process all variables referenced by global environment
- f_globals = {}
- for var in func_global_refs:
- if var in func.__globals__:
- f_globals[var] = func.__globals__[var]
-
- # defaults requires no processing
- defaults = func.__defaults__
-
- # process closure
- closure = (
- list(map(_get_cell_contents, func.__closure__))
- if func.__closure__ is not None
- else None
- )
-
- # save the dict
- dct = func.__dict__
-
- # base_globals represents the future global namespace of func at
- # unpickling time. Looking it up and storing it in globals_ref allow
- # functions sharing the same globals at pickling time to also
- # share them once unpickled, at one condition: since globals_ref is
- # an attribute of a Cloudpickler instance, and that a new CloudPickler is
- # created each time pickle.dump or pickle.dumps is called, functions
- # also need to be saved within the same invokation of
- # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There
- # is no such limitation when using Cloudpickler.dump, as long as the
- # multiple invokations are bound to the same Cloudpickler.
- base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
-
- if base_globals == {}:
- # Add module attributes used to resolve relative imports
- # instructions inside func.
- for k in ["__package__", "__name__", "__path__", "__file__"]:
- # Some built-in functions/methods such as object.__new__ have
- # their __globals__ set to None in PyPy
- if func.__globals__ is not None and k in func.__globals__:
- base_globals[k] = func.__globals__[k]
-
- return (code, f_globals, defaults, closure, dct, base_globals)
-
- def save_getset_descriptor(self, obj):
- return self.save_reduce(getattr, (obj.__objclass__, obj.__name__))
-
- dispatch[types.GetSetDescriptorType] = save_getset_descriptor
-
- def save_global(self, obj, name=None, pack=struct.pack):
- """
- Save a "global".
- The name of this method is somewhat misleading: all types get
- dispatched here.
- """
- if obj is type(None):
- return self.save_reduce(type, (None,), obj=obj)
- elif obj is type(Ellipsis):
- return self.save_reduce(type, (Ellipsis,), obj=obj)
- elif obj is type(NotImplemented):
- return self.save_reduce(type, (NotImplemented,), obj=obj)
- elif obj in _BUILTIN_TYPE_NAMES:
- return self.save_reduce(
- _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
-
- if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch
- # Parametrized typing constructs in Python < 3.7 are not compatible
- # with type checks and ``isinstance`` semantics. For this reason,
- # it is easier to detect them using a duck-typing-based check
- # (``_is_parametrized_type_hint``) than to populate the Pickler's
- # dispatch with type-specific savers.
- self._save_parametrized_type_hint(obj)
- elif name is not None:
- Pickler.save_global(self, obj, name=name)
- elif not _is_importable_by_name(obj, name=name):
- self.save_dynamic_class(obj)
- else:
- Pickler.save_global(self, obj, name=name)
-
- dispatch[type] = save_global
-
- def save_instancemethod(self, obj):
- # Memoization rarely is ever useful due to python bounding
- if obj.__self__ is None:
- self.save_reduce(getattr, (obj.im_class, obj.__name__))
- else:
- self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj)
-
- dispatch[types.MethodType] = save_instancemethod
-
- def save_property(self, obj):
- # properties not correctly saved in python
- self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__),
- obj=obj)
-
- dispatch[property] = save_property
-
- def save_classmethod(self, obj):
- orig_func = obj.__func__
- self.save_reduce(type(obj), (orig_func,), obj=obj)
-
- dispatch[classmethod] = save_classmethod
- dispatch[staticmethod] = save_classmethod
-
- def save_itemgetter(self, obj):
- """itemgetter serializer (needed for namedtuple support)"""
- class Dummy:
- def __getitem__(self, item):
- return item
- items = obj(Dummy())
- if not isinstance(items, tuple):
- items = (items,)
- return self.save_reduce(operator.itemgetter, items)
-
- if type(operator.itemgetter) is type:
- dispatch[operator.itemgetter] = save_itemgetter
-
- def save_attrgetter(self, obj):
- """attrgetter serializer"""
- class Dummy(object):
- def __init__(self, attrs, index=None):
- self.attrs = attrs
- self.index = index
- def __getattribute__(self, item):
- attrs = object.__getattribute__(self, "attrs")
- index = object.__getattribute__(self, "index")
- if index is None:
- index = len(attrs)
- attrs.append(item)
- else:
- attrs[index] = ".".join([attrs[index], item])
- return type(self)(attrs, index)
- attrs = []
- obj(Dummy(attrs))
- return self.save_reduce(operator.attrgetter, tuple(attrs))
-
- if type(operator.attrgetter) is type:
- dispatch[operator.attrgetter] = save_attrgetter
-
- def save_file(self, obj):
- """Save a file"""
-
- if not hasattr(obj, 'name') or not hasattr(obj, 'mode'):
- raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
- if obj is sys.stdout:
- return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
- if obj is sys.stderr:
- return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
- if obj is sys.stdin:
- raise pickle.PicklingError("Cannot pickle standard input")
- if obj.closed:
- raise pickle.PicklingError("Cannot pickle closed files")
- if hasattr(obj, 'isatty') and obj.isatty():
- raise pickle.PicklingError("Cannot pickle files that map to tty objects")
- if 'r' not in obj.mode and '+' not in obj.mode:
- raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)
-
- name = obj.name
-
- # TODO: also support binary mode files with io.BytesIO
- retval = io.StringIO()
-
- try:
- # Read the whole file
- curloc = obj.tell()
- obj.seek(0)
- contents = obj.read()
- obj.seek(curloc)
- except IOError:
- raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
- retval.write(contents)
- retval.seek(curloc)
-
- retval.name = name
- self.save(retval)
- self.memoize(obj)
-
- def save_ellipsis(self, obj):
- self.save_reduce(_gen_ellipsis, ())
-
- def save_not_implemented(self, obj):
- self.save_reduce(_gen_not_implemented, ())
-
- dispatch[io.TextIOWrapper] = save_file
- dispatch[type(Ellipsis)] = save_ellipsis
- dispatch[type(NotImplemented)] = save_not_implemented
-
- def save_weakset(self, obj):
- self.save_reduce(weakref.WeakSet, (list(obj),))
-
- dispatch[weakref.WeakSet] = save_weakset
-
- def save_logger(self, obj):
- self.save_reduce(logging.getLogger, (obj.name,), obj=obj)
-
- dispatch[logging.Logger] = save_logger
-
- def save_root_logger(self, obj):
- self.save_reduce(logging.getLogger, (), obj=obj)
-
- dispatch[logging.RootLogger] = save_root_logger
-
- if hasattr(types, "MappingProxyType"): # pragma: no branch
- def save_mappingproxy(self, obj):
- self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj)
-
- dispatch[types.MappingProxyType] = save_mappingproxy
-
- """Special functions for Add-on libraries"""
- def inject_addons(self):
- """Plug in system. Register additional pickling functions if modules already loaded"""
- pass
-
- if sys.version_info < (3, 7): # pragma: no branch
- def _save_parametrized_type_hint(self, obj):
- # The distorted type check sematic for typing construct becomes:
- # ``type(obj) is type(TypeHint)``, which means "obj is a
- # parametrized TypeHint"
- if type(obj) is type(Literal): # pragma: no branch
- initargs = (Literal, obj.__values__)
- elif type(obj) is type(Final): # pragma: no branch
- initargs = (Final, obj.__type__)
- elif type(obj) is type(ClassVar):
- initargs = (ClassVar, obj.__type__)
- elif type(obj) is type(Generic):
- parameters = obj.__parameters__
- if len(obj.__parameters__) > 0:
- # in early Python 3.5, __parameters__ was sometimes
- # preferred to __args__
- initargs = (obj.__origin__, parameters)
- else:
- initargs = (obj.__origin__, obj.__args__)
- elif type(obj) is type(Union):
- if sys.version_info < (3, 5, 3): # pragma: no cover
- initargs = (Union, obj.__union_params__)
- else:
- initargs = (Union, obj.__args__)
- elif type(obj) is type(Tuple):
- if sys.version_info < (3, 5, 3): # pragma: no cover
- initargs = (Tuple, obj.__tuple_params__)
- else:
- initargs = (Tuple, obj.__args__)
- elif type(obj) is type(Callable):
- if sys.version_info < (3, 5, 3): # pragma: no cover
- args = obj.__args__
- result = obj.__result__
- if args != Ellipsis:
- if isinstance(args, tuple):
- args = list(args)
- else:
- args = [args]
- else:
- (*args, result) = obj.__args__
- if len(args) == 1 and args[0] is Ellipsis:
- args = Ellipsis
- else:
- args = list(args)
- initargs = (Callable, (args, result))
- else: # pragma: no cover
- raise pickle.PicklingError(
- "Cloudpickle Error: Unknown type {}".format(type(obj))
- )
- self.save_reduce(_create_parametrized_type_hint, initargs, obj=obj)
-
-
-# Tornado support
-
-def is_tornado_coroutine(func):
- """
- Return whether *func* is a Tornado coroutine function.
- Running coroutines are not supported.
- """
- if 'tornado.gen' not in sys.modules:
- return False
- gen = sys.modules['tornado.gen']
- if not hasattr(gen, "is_coroutine_function"):
- # Tornado version is too old
- return False
- return gen.is_coroutine_function(func)
-
-
-def _rebuild_tornado_coroutine(func):
- from tornado import gen
- return gen.coroutine(func)
-
-
-# Shorthands for legacy support
-
-def dump(obj, file, protocol=None):
- """Serialize obj as bytes streamed into file
- protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
- pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
- between processes running the same Python version.
- Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
- compatibility with older versions of Python.
- """
- CloudPickler(file, protocol=protocol).dump(obj)
-
-
-def dumps(obj, protocol=None):
- """Serialize obj as a string of bytes allocated in memory
- protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
- pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
- between processes running the same Python version.
- Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
- compatibility with older versions of Python.
- """
- file = BytesIO()
- try:
- cp = CloudPickler(file, protocol=protocol)
- cp.dump(obj)
- return file.getvalue()
- finally:
- file.close()
-
-
-# including pickles unloading functions in this namespace
-load = pickle.load
-loads = pickle.loads
-
-
-# hack for __import__ not working as desired
-def subimport(name):
- __import__(name)
- return sys.modules[name]
-
-
-def dynamic_subimport(name, vars):
- mod = types.ModuleType(name)
- mod.__dict__.update(vars)
- mod.__dict__['__builtins__'] = builtins.__dict__
- return mod
-
-
-def _gen_ellipsis():
- return Ellipsis
-
-
-def _gen_not_implemented():
- return NotImplemented
-
-
-def _get_cell_contents(cell):
- try:
- return cell.cell_contents
- except ValueError:
- # sentinel used by ``_fill_function`` which will leave the cell empty
- return _empty_cell_value
-
-
-def instance(cls):
- """Create a new instance of a class.
- Parameters
- ----------
- cls : type
- The class to create an instance of.
- Returns
- -------
- instance : cls
- A new instance of ``cls``.
- """
- return cls()
-
-
-@instance
-class _empty_cell_value(object):
- """sentinel for empty closures
- """
- @classmethod
- def __reduce__(cls):
- return cls.__name__
-
-
-def _fill_function(*args):
- """Fills in the rest of function data into the skeleton function object
- The skeleton itself is create by _make_skel_func().
- """
- if len(args) == 2:
- func = args[0]
- state = args[1]
- elif len(args) == 5:
- # Backwards compat for cloudpickle v0.4.0, after which the `module`
- # argument was introduced
- func = args[0]
- keys = ['globals', 'defaults', 'dict', 'closure_values']
- state = dict(zip(keys, args[1:]))
- elif len(args) == 6:
- # Backwards compat for cloudpickle v0.4.1, after which the function
- # state was passed as a dict to the _fill_function it-self.
- func = args[0]
- keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
- state = dict(zip(keys, args[1:]))
- else:
- raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
-
- # - At pickling time, any dynamic global variable used by func is
- # serialized by value (in state['globals']).
- # - At unpickling time, func's __globals__ attribute is initialized by
- # first retrieving an empty isolated namespace that will be shared
- # with other functions pickled from the same original module
- # by the same CloudPickler instance and then updated with the
- # content of state['globals'] to populate the shared isolated
- # namespace with all the global variables that are specifically
- # referenced for this function.
- func.__globals__.update(state['globals'])
-
- func.__defaults__ = state['defaults']
- func.__dict__ = state['dict']
- if 'annotations' in state:
- func.__annotations__ = state['annotations']
- if 'doc' in state:
- func.__doc__ = state['doc']
- if 'name' in state:
- func.__name__ = state['name']
- if 'module' in state:
- func.__module__ = state['module']
- if 'qualname' in state:
- func.__qualname__ = state['qualname']
- if 'kwdefaults' in state:
- func.__kwdefaults__ = state['kwdefaults']
- # _cloudpickle_subimports is a set of submodules that must be loaded for
- # the pickled function to work correctly at unpickling time. Now that these
- # submodules are depickled (hence imported), they can be removed from the
- # object's state (the object state only served as a reference holder to
- # these submodules)
- if '_cloudpickle_submodules' in state:
- state.pop('_cloudpickle_submodules')
-
- cells = func.__closure__
- if cells is not None:
- for cell, value in zip(cells, state['closure_values']):
- if value is not _empty_cell_value:
- cell_set(cell, value)
-
- return func
-
-
-def _make_empty_cell():
- if False:
- # trick the compiler into creating an empty cell in our lambda
- cell = None
- raise AssertionError('this route should not be executed')
-
- return (lambda: cell).__closure__[0]
-
-
-def _make_skel_func(code, cell_count, base_globals=None):
- """ Creates a skeleton function object that contains just the provided
- code and the correct number of cells in func_closure. All other
- func attributes (e.g. func_globals) are empty.
- """
- # This is backward-compatibility code: for cloudpickle versions between
- # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
- # should now always be a dictionary.
- if base_globals is None or isinstance(base_globals, str):
- base_globals = {}
-
- base_globals['__builtins__'] = __builtins__
-
- closure = (
- tuple(_make_empty_cell() for _ in range(cell_count))
- if cell_count >= 0 else
- None
- )
- return types.FunctionType(code, base_globals, None, None, closure)
-
-
-def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
- class_tracker_id, extra):
- """Build dynamic class with an empty __dict__ to be filled once memoized
- If class_tracker_id is not None, try to lookup an existing class definition
- matching that id. If none is found, track a newly reconstructed class
- definition under that id so that other instances stemming from the same
- class id will also reuse this class definition.
- The "extra" variable is meant to be a dict (or None) that can be used for
- forward compatibility shall the need arise.
- """
- skeleton_class = types.new_class(
- name, bases, {'metaclass': type_constructor},
- lambda ns: ns.update(type_kwargs)
- )
- return _lookup_class_or_track(class_tracker_id, skeleton_class)
-
-
-def _rehydrate_skeleton_class(skeleton_class, class_dict):
- """Put attributes from `class_dict` back on `skeleton_class`.
- See CloudPickler.save_dynamic_class for more info.
- """
- registry = None
- for attrname, attr in class_dict.items():
- if attrname == "_abc_impl":
- registry = attr
- else:
- setattr(skeleton_class, attrname, attr)
- if registry is not None:
- for subclass in registry:
- skeleton_class.register(subclass)
-
- return skeleton_class
-
-
-def _make_skeleton_enum(bases, name, qualname, members, module,
- class_tracker_id, extra):
- """Build dynamic enum with an empty __dict__ to be filled once memoized
- The creation of the enum class is inspired by the code of
- EnumMeta._create_.
- If class_tracker_id is not None, try to lookup an existing enum definition
- matching that id. If none is found, track a newly reconstructed enum
- definition under that id so that other instances stemming from the same
- class id will also reuse this enum definition.
- The "extra" variable is meant to be a dict (or None) that can be used for
- forward compatibility shall the need arise.
- """
- # enums always inherit from their base Enum class at the last position in
- # the list of base classes:
- enum_base = bases[-1]
- metacls = enum_base.__class__
- classdict = metacls.__prepare__(name, bases)
-
- for member_name, member_value in members.items():
- classdict[member_name] = member_value
- enum_class = metacls.__new__(metacls, name, bases, classdict)
- enum_class.__module__ = module
- enum_class.__qualname__ = qualname
-
- return _lookup_class_or_track(class_tracker_id, enum_class)
-
-
-def _is_dynamic(module):
- """
- Return True if the module is special module that cannot be imported by its
- name.
- """
- # Quick check: module that have __file__ attribute are not dynamic modules.
- if hasattr(module, '__file__'):
- return False
-
- if module.__spec__ is not None:
- return False
-
- # In PyPy, Some built-in modules such as _codecs can have their
- # __spec__ attribute set to None despite being imported. For such
- # modules, the ``_find_spec`` utility of the standard library is used.
- parent_name = module.__name__.rpartition('.')[0]
- if parent_name: # pragma: no cover
- # This code handles the case where an imported package (and not
- # module) remains with __spec__ set to None. It is however untested
- # as no package in the PyPy stdlib has __spec__ set to None after
- # it is imported.
- try:
- parent = sys.modules[parent_name]
- except KeyError:
- msg = "parent {!r} not in sys.modules"
- raise ImportError(msg.format(parent_name))
- else:
- pkgpath = parent.__path__
- else:
- pkgpath = None
- return _find_spec(module.__name__, pkgpath, module) is None
-
-
-def _make_typevar(name, bound, constraints, covariant, contravariant,
- class_tracker_id):
- tv = typing.TypeVar(
- name, *constraints, bound=bound,
- covariant=covariant, contravariant=contravariant
- )
- if class_tracker_id is not None:
- return _lookup_class_or_track(class_tracker_id, tv)
- else: # pragma: nocover
- # Only for Python 3.5.3 compat.
- return tv
-
-
-def _decompose_typevar(obj):
- try:
- class_tracker_id = _get_or_create_tracker_id(obj)
- except TypeError: # pragma: nocover
- # TypeVar instances are not weakref-able in Python 3.5.3
- class_tracker_id = None
- return (
- obj.__name__, obj.__bound__, obj.__constraints__,
- obj.__covariant__, obj.__contravariant__,
- class_tracker_id,
- )
-
-
-def _typevar_reduce(obj):
- # TypeVar instances have no __qualname__ hence we pass the name explicitly.
- module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__)
- if module_and_name is None:
- return (_make_typevar, _decompose_typevar(obj))
- return (getattr, module_and_name)
-
-
-def _get_bases(typ):
- if hasattr(typ, '__orig_bases__'):
- # For generic types (see PEP 560)
- bases_attr = '__orig_bases__'
- else:
- # For regular class objects
- bases_attr = '__bases__'
- return getattr(typ, bases_attr)
diff --git a/heronpy/api/serializer.py b/heronpy/api/serializer.py
index dc55456..e5e5568 100644
--- a/heronpy/api/serializer.py
+++ b/heronpy/api/serializer.py
@@ -26,7 +26,7 @@ try:
except:
import pickle
-import heronpy.api.cloudpickle as cloudpickle
+import cloudpickle
class IHeronSerializer:
"""Serializer interface for Heron"""
diff --git a/licenses/LICENSE-cloudpickle.txt b/licenses/LICENSE-cloudpickle.txt
deleted file mode 100644
index 4582f84..0000000
--- a/licenses/LICENSE-cloudpickle.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-This module was extracted from the `cloud` package, developed by
-PiCloud, Inc.
-
-Copyright (c) 2015, Cloudpickle contributors.
-Copyright (c) 2012, Regents of the University of California.
-Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- * Neither the name of the University of California, Berkeley nor the
- names of its contributors may be used to endorse or promote
- products derived from this software without specific prior written
- permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
-TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/scripts/packages/heronpy/__apiinit__.py.template b/scripts/packages/heronpy/__apiinit__.py.template
index e6578c3..8b48604 100644
--- a/scripts/packages/heronpy/__apiinit__.py.template
+++ b/scripts/packages/heronpy/__apiinit__.py.template
@@ -36,7 +36,6 @@ __all__ = [
'global_metrics',
'metrics',
'serializer',
- 'cloudpickle',
'state',
'spout',
'stream',
diff --git a/scripts/packages/heronpy/requirements.txt b/scripts/packages/heronpy/requirements.txt
index 82c7699..6ce5b89 100644
--- a/scripts/packages/heronpy/requirements.txt
+++ b/scripts/packages/heronpy/requirements.txt
@@ -1 +1,2 @@
protobuf==3.8.0
+cloudpickle~=1.5.0