From 53a9a72a4f12251499423fb81703d1b174158f99 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 10 Nov 2020 14:06:17 -0600 Subject: [PATCH 01/14] delete sweet_pickle subpackage and references to it. Update its use in apptools.naming to use apptools.persistence or pickle from the standard library instead --- README.rst | 3 - apptools/naming/object_serializer.py | 13 +- apptools/persistence/versioned_unpickler.py | 4 +- apptools/sweet_pickle/README.txt | 8 - apptools/sweet_pickle/__init__.py | 168 ------ apptools/sweet_pickle/global_registry.py | 99 ---- apptools/sweet_pickle/placeholder.py | 25 - apptools/sweet_pickle/tests/__init__.py | 1 - .../tests/class_mapping_classes.py | 21 - .../tests/state_function_classes.py | 55 -- .../sweet_pickle/tests/test_class_mapping.py | 99 ---- .../tests/test_global_registry.py | 76 --- .../sweet_pickle/tests/test_state_function.py | 157 ------ .../tests/test_two_stage_unpickler.py | 184 ------- apptools/sweet_pickle/tests/test_updater.py | 322 ----------- apptools/sweet_pickle/updater.py | 300 ----------- apptools/sweet_pickle/versioned_unpickler.py | 507 ------------------ 17 files changed, 7 insertions(+), 2035 deletions(-) delete mode 100644 apptools/sweet_pickle/README.txt delete mode 100644 apptools/sweet_pickle/__init__.py delete mode 100644 apptools/sweet_pickle/global_registry.py delete mode 100644 apptools/sweet_pickle/placeholder.py delete mode 100644 apptools/sweet_pickle/tests/__init__.py delete mode 100644 apptools/sweet_pickle/tests/class_mapping_classes.py delete mode 100644 apptools/sweet_pickle/tests/state_function_classes.py delete mode 100644 apptools/sweet_pickle/tests/test_class_mapping.py delete mode 100644 apptools/sweet_pickle/tests/test_global_registry.py delete mode 100644 apptools/sweet_pickle/tests/test_state_function.py delete mode 100644 apptools/sweet_pickle/tests/test_two_stage_unpickler.py delete mode 100644 apptools/sweet_pickle/tests/test_updater.py delete mode 100644 apptools/sweet_pickle/updater.py delete mode 100644 apptools/sweet_pickle/versioned_unpickler.py diff --git a/README.rst b/README.rst index 8b5bb6b63..29f3e9624 100644 --- a/README.rst +++ b/README.rst @@ -30,9 +30,6 @@ that is commonly needed by many applications listener of selected items in an application. - **apptools.scripting**: A framework for automatic recording of Python scripts. -- **apptools.sweet_pickle**: Handles class-level versioning, to support - loading of saved data that exist over several generations of internal class - structures. - **apptools.type_manager**: Manages type extensions, including factories to generate adapters, and hooks for methods and functions. - **apptools.undo**: Supports undoing and scripting application commands. diff --git a/apptools/naming/object_serializer.py b/apptools/naming/object_serializer.py index 4c5578418..a3d8cbac4 100644 --- a/apptools/naming/object_serializer.py +++ b/apptools/naming/object_serializer.py @@ -18,11 +18,10 @@ import logging from traceback import print_exc from os.path import splitext -#import cPickle -#import pickle +import pickle # Enthought library imports. -import apptools.sweet_pickle as sweet_pickle +from apptools.persistence.versioned_unpickler import VersionedUnpickler from traits.api import HasTraits, Str @@ -56,9 +55,7 @@ def load(self, path): f = open(path, 'rb') try: try: - obj = sweet_pickle.load(f) -# obj = cPickle.load(f) -# obj = pickle.load(f) + obj = VersionedUnpickler(f).load() except Exception as ex: print_exc() logger.exception( "Failed to load pickle file: %s, %s" % (path, ex)) @@ -86,9 +83,7 @@ def save(self, path, obj): # Pickle the object. f = open(actual_path, 'wb') try: - sweet_pickle.dump(obj, f, 1) -# cPickle.dump(obj, f, 1) -# pickle.dump(obj, f, 1) + pickle.dump(obj, f, 1) except Exception as ex: logger.exception( "Failed to pickle into file: %s, %s, object:%s" % (path, ex, obj)) diff --git a/apptools/persistence/versioned_unpickler.py b/apptools/persistence/versioned_unpickler.py index 36d6ea164..082953192 100644 --- a/apptools/persistence/versioned_unpickler.py +++ b/apptools/persistence/versioned_unpickler.py @@ -1,5 +1,6 @@ # Standard library imports -from pickle import * +from pickle import _Unpickler as Unpickler +from pickle import UnpicklingError, BUILD import sys import logging from types import GeneratorType, MethodType @@ -96,6 +97,7 @@ def initialize(self, max_pass): # point, we would have the dispatch still pointing to # NewPickler.load_build whereas the object being passed in will be an # Unpickler instance, causing a TypeError. + @classmethod def load_build(cls, obj): # Just save the instance in the list of objects. if isinstance(obj, NewUnpickler): diff --git a/apptools/sweet_pickle/README.txt b/apptools/sweet_pickle/README.txt deleted file mode 100644 index f083411e1..000000000 --- a/apptools/sweet_pickle/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -The sweet_pickle package is not in wide use and thus has not been fully -vetted. As such, we reserve the right to make significant refactorings -to this package, classes, and API. - -**** USE ONLY IF YOU ACCEPT THIS SITUATION. **** - -See the comments in the package init for an explanation of why this package -exists. diff --git a/apptools/sweet_pickle/__init__.py b/apptools/sweet_pickle/__init__.py deleted file mode 100644 index e7a9037ba..000000000 --- a/apptools/sweet_pickle/__init__.py +++ /dev/null @@ -1,168 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -""" A pickle framework that supports unpickling of refactored versions of - old classes. Part of the AppTools project of the Enthought Tool Suite. - - Beyond refactoring support, there are additional useful capabilities of - this package: - - - HasTraits objects are fully pickled when sweet_pickle.dump() or - sweet_pickle.dumps() are called. This means every trait's value - is pickled explicitly. This allows improved self-consistency as - values within unpickled objects correspond to the default trait - values of the instance that was pickled rather than the default - values of the current class definition. - - Multiple updaters can be registered to update state for any version - of a class. This is extremely useful when an instance being - pickled and unpickled is the result of an addition of one or more - Trait Categories to a class. In that case, the Category contributor - can also register a state function to update the Category's traits - through versioning. - - - We have duplicated the API of the Python 'pickle' module within this - package. Thus, users can simply import sweet_pickle in places where - they previously used pickle or cPickle. For example:: - - import cPickle ---> import apptools.sweet_pickle as pickle - s = pickle.dumps(obj) s = pickle.dumps(obj) - pickle.loads(s) pickle.loads(s) - - Pickles generated by this package *can be* unpickled by standard pickle - or cPickle, though you lose the benefit of the refactoring support during - unpickling. - - As a result of the above, this package is a drop-in replacement for the - Python 'pickle' module or 'cPickle' module and should be safe to use even - if you never version or refactor your own classes. In fact, we STRONGLY - RECOMMEND that you use this framework for all of your pickle needs because - you never know when one of the classes you encounter during unpickling has - been versioned or otherwise refactored by someone else. - - See module 'pickle' for more basic information about pickling. - - - The most common way to benefit from the versioning capabilities of this - framework is to register class mappings and state modification functions - with the global updater registry (more detail is below.) However, you may - also choose to explicitly instantiate an Unpickler and provide it with your - own explicit definition of class mappings and state modification functions. - We do not provide any help on the latter at this time. - - You can register class mappings and state modification functions with the - global updater registry using the methods provided on the Updater instance - that is the global registry. You can get this instance by calling the - 'get_global_registry' function exposed through this package's namespace. - - This framework has been designed so that you can register your class - mappings and state modification functions during the import of the module - or package that contained the original class. However, you may also - register your mappings and functions at any point such that the they are - known to the framework prior to, or become known during the attempt to, - unpickle the class they modify. - - The framework will call a __setstate__ method on the final target class - of any unpickled instance. It will not call __setstate__ methods on any - beginning or intermediate classes within a chain of class mappings. - - - A class mapping is used to redirect the unpickling of one class to return - an instantiation of another class. The classes can be in different - modules, and the modules in different packages. Mappings can be chained. - For example, given the mappings:: - - foo.bar.Bar --> foo.baz.Baz - foo.baz.Baz --> foo.Foo - - An attempt to unpickle a foo.bar.Bar would actually generate a foo.Foo - instance. - - A state modification function is called during the execution of the - __setstate__ method during unpickling of an object of the type and version - for which it was registered for. The function must accept a single - argument which is a state dictionary and must then return the modified - state dictionary. Additionally, the function should change the version - variable within the state to represent the version the new state - represents. (The framework will assume an increment of one if this is - not done.) The framework ensures that state modification functions - are chained appropriately to convert through multiple versions and/or class - mappings. - - Note that refactorings that cause classes to be completed removed from - the source code can be supported, without breaking unpickling of object - hierarchies that include an instace of that class, by adding a mapping - to the Placeholder class in the placeholder module. -""" - -############################################################################## -# Implement the Python pickle package API -############################################################################## - -# Expose our custom pickler as the standard Unpickler -from .versioned_unpickler import VersionedUnpickler as Unpickler - -# Use our custom unpickler to load from files -def load(file, max_pass=-1): - return Unpickler(file).load(max_pass) - -# Use our custom unpickler to load from strings -def loads(str, max_pass=-1): - from io import BytesIO - file = BytesIO(str) - - return Unpickler(file).load(max_pass) - -# We don't customize the Python pickler, though we do use the cPickle module -# for improved performance. -from six.moves.cPickle import Pickler - -# Implement the dump and dumps methods so that all traits in a HasTraits object -# get included in the pickle. -def dump(obj, file, protocol=2): - _flush_traits(obj) - from six.moves.cPickle import dump as d - return d(obj, file, protocol) - -def dumps(obj, protocol=2): - _flush_traits(obj) - from six.moves.cPickle import dumps as ds - return ds(obj, protocol) - -# We don't customize exceptions so just map to the Python pickle package -from pickle import PickleError, PicklingError, UnpicklingError - - -############################################################################## -# Allow retrieval of the global registry -############################################################################## - -from .global_registry import get_global_registry - - -############################################################################## -# Expose our Updater class so users can explicitly create their own. -############################################################################## - -from .updater import Updater - - -############################################################################## -# Method to ensure that all traits on a has traits object are included in a -# pickle. -############################################################################## - -def _flush_traits(obj): - if hasattr(obj, 'trait_names'): - for name, value in obj.traits().items(): - if value.type == 'trait': - try: - getattr(obj, name) - except AttributeError: - pass diff --git a/apptools/sweet_pickle/global_registry.py b/apptools/sweet_pickle/global_registry.py deleted file mode 100644 index 3236fcfd2..000000000 --- a/apptools/sweet_pickle/global_registry.py +++ /dev/null @@ -1,99 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -""" Manages a singleton updater that acts as a global registry. - - Our goal is to enable the apptools.sweet_pickle framework to - understand how pickled data should be treated during unpickling so - that the resulting object hierarchy reflects the current versions - of the object's classes -AND- that this should work no matter who - is doing the unpickling. This requires a global registry since - there is no way to know in advance what objects are contained within - a given pickle file, and thus no way to gather the required - information to know how to treat the pickle data during unpickling. - - For example, a pickle of an Envisage project may contain many - custom classes, instantiated at the behest of various plugins, - which have gone through various versionings and refactorings. But - it is the project plugin that needs to unpickle these objects to - 'load' a project, not the other plugins that added those custom - class instances into the project. - - - This registry is used by the apptools.sweet_pickle framework's - unpickler only by default. That is, only if no updater was - explicitly provided. - - - It is important that users interact with the registry through the - provided methods. If they do not, then the reference they receive - will be the one that was in place at the time of the import which - may or MAY NOT be the current repository due to the way this - framework manages the repository. -""" - -try: - import six.moves._thread as _thread -except ImportError: - import six.moves._dummy_thread as _thread - - -############################################################################## -# function 'get_global_registry' -############################################################################## - -def get_global_registry(): - """ Returns the global registry in a manner that allows for lazy - instantiation. - """ - global _global_registry, _global_registry_lock - - # Do we need to create the registry? - if _global_registry is None: - - # We can only do this safely in a threaded situation by using a lock. - # Note that the previous check for None doesn't guarantee we are - # the only one trying to create an instance, so, we'll check for none - # again once we acquire the lock and then only create the singleton - # if there still isn't an instance. - _global_registry_lock.acquire() - try: - if _global_registry is None: - from .updater import Updater - _global_registry = Updater() - finally: - _global_registry_lock.release() - - return _global_registry - - -############################################################################## -# private function '_clear_global_registry' -############################################################################## - -def _clear_global_registry(): - """ Clears out the current global registry. - - This exists purely to allow testing of the global registry and the - apptools.sweet_pickle framework. THIS METHOD SHOULD NEVER BE - CALLED DURING NORMAL OPERATIONS! - """ - global _global_registry - _global_registry = None - - -############################################################################## -# private, but global, variables -############################################################################## - -# The global singleton updater -_global_registry = None - -# The lock used to make access to the global singleton thread safe -_global_registry_lock = _thread.allocate_lock() diff --git a/apptools/sweet_pickle/placeholder.py b/apptools/sweet_pickle/placeholder.py deleted file mode 100644 index a388a1f03..000000000 --- a/apptools/sweet_pickle/placeholder.py +++ /dev/null @@ -1,25 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Vibha Srinivasan -# -#----------------------------------------------------------------------------- - -""" An empty class that serves as a placeholder to map a class to when - that class has been deleted as a result of a refactoring. -""" - -# Enthought library imports -from traits.api import HasTraits - - -############################################################################## -# class 'PlaceHolder' -############################################################################## - -class PlaceHolder(HasTraits): - """ An empty class that serves as a placeholder to map a class to when - that class has been deleted as a result of a refactoring. - """ diff --git a/apptools/sweet_pickle/tests/__init__.py b/apptools/sweet_pickle/tests/__init__.py deleted file mode 100644 index 8b1378917..000000000 --- a/apptools/sweet_pickle/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/apptools/sweet_pickle/tests/class_mapping_classes.py b/apptools/sweet_pickle/tests/class_mapping_classes.py deleted file mode 100644 index 4b50ca9b9..000000000 --- a/apptools/sweet_pickle/tests/class_mapping_classes.py +++ /dev/null @@ -1,21 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -############################################################################## -# Classes to use within the tests -############################################################################## - -class Foo: - pass - -class Bar: - pass - -class Baz: - pass diff --git a/apptools/sweet_pickle/tests/state_function_classes.py b/apptools/sweet_pickle/tests/state_function_classes.py deleted file mode 100644 index 9a7911fc6..000000000 --- a/apptools/sweet_pickle/tests/state_function_classes.py +++ /dev/null @@ -1,55 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -# Standard library imports -import logging - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry -from traits.api import Bool, Float, HasTraits, Int, Str - - -logger = logging.getLogger(__name__) - - -############################################################################## -# Classes to use within the tests -############################################################################## - -class Foo(HasTraits): - _enthought_pickle_version = Int(1) - b1 = Bool(False) - f1 = Float(1) - i1 = Int(1) - s1 = Str('foo') - -class Bar(HasTraits): - _enthought_pickle_version = Int(2) - b2 = Bool(True) - f2 = Float(2) - i2 = Int(2) - s2 = Str('bar') - -class Baz(HasTraits): - _enthought_pickle_version = Int(3) - b3 = Bool(False) - f3 = Float(3) - i3 = Int(3) - s3 = Str('baz') - def __setstate__(self, state): - logger.debug('Running Baz\'s original __setstate__') - if state['_enthought_pickle_version'] < 3: - info = [('b2', 'b3'), ('f2', 'f3'), ('i2', 'i3'), ('s2', 's3')] - for old, new in info: - if old in state: - state[new] = state[old] - del state[old] - state['_enthought_pickle_version'] = 3 - self.__dict__.update(state) diff --git a/apptools/sweet_pickle/tests/test_class_mapping.py b/apptools/sweet_pickle/tests/test_class_mapping.py deleted file mode 100644 index 0b5f847de..000000000 --- a/apptools/sweet_pickle/tests/test_class_mapping.py +++ /dev/null @@ -1,99 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -""" Tests the class mapping functionality of the enthought.pickle - framework. -""" - -# Standard library imports. -import unittest - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry - - -############################################################################## -# Classes to use within the tests -############################################################################## - -# Need complete package name so that mapping matches correctly. -# The problem here is the Python loader that will load the same module with -# multiple names in sys.modules due to relative naming. Nice. -from apptools.sweet_pickle.tests.class_mapping_classes import Foo, Bar, Baz - -############################################################################## -# class 'ClassMappingTestCase' -############################################################################## - -class ClassMappingTestCase(unittest.TestCase): - """ Tests the class mapping functionality of the apptools.sweet_pickle - framework. - """ - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """ Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - - ########################################################################## - # 'ClassMappingTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_infinite_loop_detection(self): - """ Validates that the class mapping framework detects infinite - loops of class mappings. - """ - # Add mappings to the registry - self.registry.add_mapping_to_class(Foo.__module__, Foo.__name__, Bar) - self.registry.add_mapping_to_class(Bar.__module__, Bar.__name__, Baz) - self.registry.add_mapping_to_class(Baz.__module__, Baz.__name__, Foo) - - # Validate that an exception is raised when trying to unpickle an - # instance anywhere within the circular definition. - def fn(o): - sweet_pickle.loads(sweet_pickle.dumps(o)) - self.assertRaises(sweet_pickle.UnpicklingError, fn, Foo()) - self.assertRaises(sweet_pickle.UnpicklingError, fn, Bar()) - self.assertRaises(sweet_pickle.UnpicklingError, fn, Baz()) - - - def test_unpickled_class_mapping(self): - - # Add the mappings to the registry - self.registry.add_mapping_to_class(Foo.__module__, Foo.__name__, Bar) - self.registry.add_mapping_to_class(Bar.__module__, Bar.__name__, Baz) - - # Validate that unpickling the first class gives us an instance of - # the third class. - start = Foo() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - - # Validate that unpickling the second class gives us an instance of - # the third class. - start = Bar() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) diff --git a/apptools/sweet_pickle/tests/test_global_registry.py b/apptools/sweet_pickle/tests/test_global_registry.py deleted file mode 100644 index af1e23cdc..000000000 --- a/apptools/sweet_pickle/tests/test_global_registry.py +++ /dev/null @@ -1,76 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -""" Tests the global registry functionality of the sweet_pickle framework. -""" - -# Standard library imports. -import unittest - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry - - -############################################################################## -# class 'GlobalRegistryTestCase' -############################################################################## - -class GlobalRegistryTestCase(unittest.TestCase): - """ Tests the global registry functionality of the sweet_pickle framework. - """ - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """ Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - - ########################################################################## - # 'GlobalRegistryTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_clearing(self): - """ Validates that clearing the registry gives us a new registry. - """ - _clear_global_registry() - self.assertNotEqual(self.registry, sweet_pickle.get_global_registry()) - - - def test_registry_starts_empty(self): - """ Validates that the registry is starting empty for each test. - """ - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - - - def test_returns_singleton(self): - """ Validates that the getter returns the same global registry - """ - # Just try it a few times. - self.assertEqual(self.registry, sweet_pickle.get_global_registry()) - self.assertEqual(self.registry, sweet_pickle.get_global_registry()) - self.assertEqual(self.registry, sweet_pickle.get_global_registry()) diff --git a/apptools/sweet_pickle/tests/test_state_function.py b/apptools/sweet_pickle/tests/test_state_function.py deleted file mode 100644 index 8e802a83b..000000000 --- a/apptools/sweet_pickle/tests/test_state_function.py +++ /dev/null @@ -1,157 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -""" Tests the state function functionality of the apptools.sweet_pickle - framework. -""" - -# Standard library imports. -import unittest -import logging - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry -from traits.api import Bool, Float, HasTraits, Int, Str - - -logger = logging.getLogger(__name__) - - -############################################################################## -# Classes to use within the tests -############################################################################## - -# Need complete package name so that mapping matches correctly. -# The problem here is the Python loader that will load the same module with -# multiple names in sys.modules due to relative naming. Nice. -from apptools.sweet_pickle.tests.state_function_classes import Foo, Bar, Baz - -############################################################################## -# State functions to use within the tests -############################################################################## - -def bar_state_function(state): - for old, new in [('b1', 'b2'), ('f1', 'f2'), ('i1', 'i2'), ('s1', 's2')]: - state[new] = state[old] - del state[old] - state['_enthought_pickle_version'] = 2 - return state - - -############################################################################## -# class 'StateFunctionTestCase' -############################################################################## - -class StateFunctionTestCase(unittest.TestCase): - """ Tests the state function functionality of the apptools.sweet_pickle - framework. - """ - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """ Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - # Add the class mappings to the registry - self.registry.add_mapping_to_class(Foo.__module__, Foo.__name__, - Bar) - self.registry.add_mapping_to_class(Bar.__module__, Bar.__name__, - Baz) - - - ########################################################################## - # 'StateFunctionTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_normal_setstate(self): - """ Validates that only existing setstate methods are called when - there are no registered state functions in the class chain. - """ - # Validate that unpickling the first class gives us an instance of - # the third class with the appropriate attribute values. It will have - # the default Foo values (because there is no state function to move - # them) and also the default Baz values (since they inherit the - # trait defaults because nothing overwrote the values.) - start = Foo() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - self._assertAttributes(end, 1, (False, 1, 1, 'foo')) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (False, 3, 3, 'baz')) - - # Validate that unpickling the second class gives us an instance of - # the third class with the appropriate attribute values. It will have - # only the Baz attributes with the Bar values (since the __setstate__ - # on Baz converted the Bar attributes to Baz attributes.) - start = Bar() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (True, 2, 2, 'bar')) - - - def test_unpickled_chain_functionality(self): - """ Validates that the registered state functions are used when - unpickling. - """ - # Add the state function to the registry - self.registry.add_state_function_for_class(Bar, 2, - bar_state_function) - - # Validate that unpickling the first class gives us an instance of - # the third class with the appropriate attribute values. - start = Foo() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - self._assertAttributes(end, 1, None) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (False, 1, 1, 'foo')) - - # Validate that unpickling the second class gives us an instance of - # the third class. - start = Bar() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (True, 2, 2, 'bar')) - - - ### protected interface ################################################## - - def _assertAttributes(self, obj, suffix, values): - """ Ensures that the specified object's attributes with the specified - suffix have the expected values. If values is None, then the - attributes shouldn't exist. - """ - attributeNames = ['b', 'f', 'i', 's'] - for i in range(len(attributeNames)): - name = attributeNames[i] + str(suffix) - if values is None: - self.assertEqual(False, hasattr(obj, name), - 'Obj [%s] has attribute [%s]' % (obj, name)) - else: - self.assertEqual(values[i], getattr(obj, name), - 'Obj [%s] attribute [%s] has [%s] instead of [%s]' % \ - (obj, name, values[i], getattr(obj, name))) diff --git a/apptools/sweet_pickle/tests/test_two_stage_unpickler.py b/apptools/sweet_pickle/tests/test_two_stage_unpickler.py deleted file mode 100644 index fdc67b8c7..000000000 --- a/apptools/sweet_pickle/tests/test_two_stage_unpickler.py +++ /dev/null @@ -1,184 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2008 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# Prabhu Ramachandran -#----------------------------------------------------------------------------- - -# Test cases. -from __future__ import print_function - -import random -import pickle -import unittest - -import apptools.sweet_pickle as sweet_pickle - -######################################## - -# Usecase1: generic case -class A(object): - def __init__(self, b=None): - self.x = 0 - self.set_b(b) - - def set_b(self, b): - self.b_ref = b - if b and hasattr(b, 'y'): - self.x = b.y - - def __setstate__(self, state): - self.__dict__.update(state) - self.set_b(self.b_ref) - - def __initialize__(self): - while self.b_ref is None and self.b_ref.y != 0: - yield True - self.set_b(self.b_ref) - - -class B(object): - def __init__(self, a=None): - self.y = 0 - self.set_a(a) - - def __getstate__(self): - state = self.__dict__.copy() - del state['y'] - return state - - def __setstate__(self, state): - self.__dict__.update(state) - self.set_a(self.a_ref) - - def set_a(self, a): - self.a_ref = a - if a and hasattr(a, 'x'): - self.y = a.x - - def __initialize__(self): - while self.a_ref is None and self.a_ref.x != 0: - yield True - self.set_a(self.a_ref) - - -class GenericTestCase(unittest.TestCase): - def test_generic(self): - print('\nRunning generic test...') - - a = A() - b = B() - a.x = random.randint(1, 100) - b.set_a(a) - a.set_b(b) - value = a.x - - # This will fail, even though we have a __setstate__ method. - s = pickle.dumps(a) - new_a = pickle.loads(s) - try: - print('\ta.x: %s' % new_a.x) - print('\ta.b_ref.y: %s' % new_a.b_ref.y) - except Exception as msg: - print('\t%s' % 'Expected Error'.center(75,'*')) - print('\t%s' % msg) - print('\t%s' % ('*'*75)) - - # This will work! - s = pickle.dumps(a) - new_a = sweet_pickle.loads(s) - assert new_a.x == new_a.b_ref.y == value - - print('Generic test succesfull.\n\n') - - -######################################## -# Usecase2: Toy Application -import re -class StringFinder(object): - def __init__(self, source, pattern): - self.pattern = pattern - self.source = source - self.data = [] - - def __getstate__(self): - s = self.__dict__.copy() - del s['data'] - return s - - def __initialize__(self): - while not self.source.initialized: - yield True - self.find() - - def find(self): - pattern = self.pattern - string = self.source.data - self.data = [(x.start(), x.end()) for x in re.finditer(pattern, string)] - - -class XMLFileReader(object): - def __init__(self, file_name): - self.data = '' - self.initialized = False - self.file_name = file_name - self.read() - - def __getstate__(self): - s = self.__dict__.copy() - del s['data'] - del s['initialized'] - return s - - def __setstate__(self, state): - self.__dict__.update(state) - self.read() - - def read(self): - # Make up random data from the filename - data = [10*x for x in self.file_name] - random.shuffle(data) - self.data = ' '.join(data) - self.initialized = True - - -class Application(object): - def __init__(self): - self.reader = XMLFileReader('some_test_file.xml') - self.finder = StringFinder(self.reader, 'e') - - def get(self): - print('\t%s' % self.finder.data) - print('\t%s' % self.reader.data) - - -class ToyAppTestCase(unittest.TestCase): - def test_toy_app(self): - print('\nRunning toy app test...') - - a = Application() - a.finder.find() - a.get() - s = pickle.dumps(a) - b = pickle.loads(s) - # Won't work. - try: - b.get() - except Exception as msg: - print('\t%s' % 'Expected Error'.center(75,'*')) - print('\t%s' % msg) - print('\t%s' % ('*'*75)) - - # Works fine. - c = sweet_pickle.loads(s) - c.get() - - print('Toy app test succesfull.\n\n') - - -if __name__ == '__main__': - test_generic() - test_toy_app() - print('ALL TESTS SUCCESFULL\n') diff --git a/apptools/sweet_pickle/tests/test_updater.py b/apptools/sweet_pickle/tests/test_updater.py deleted file mode 100644 index 946498155..000000000 --- a/apptools/sweet_pickle/tests/test_updater.py +++ /dev/null @@ -1,322 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -#----------------------------------------------------------------------------- - -""" Tests the updater functionality of the sweet_pickle framework. -""" - -# Standard library imports. -import unittest -import logging - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry - - -logger = logging.getLogger(__name__) - - -############################################################################## -# class 'UpdaterTestCase' -############################################################################## - -class UpdaterTestCase(unittest.TestCase): - """ Tests the updater functionality of the sweet_pickle framework. - """ - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """ Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - - ########################################################################## - # 'UpdaterTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_add_mapping(self): - """ Validates the behavior of the add_mapping function. - """ - # Add a single mapping and validate that all it did was add a class - # mapping and that the mapping has the expected values. - key = ('foo', 'Foo') - value = ('bar', 'Bar') - self.registry.add_mapping(key[0], key[1], value[0], value[1]) - self._validate_exactly_one_class_map(key, value) - - # Overwrite with a new mapping and validate the state is what we - # expect. - value = ('baz', 'Baz') - self.registry.add_mapping(key[0], key[1], value[0], value[1]) - self._validate_exactly_one_class_map(key, value) - - - def test_add_mapping_to_class(self): - """ Validates the behavior of the add_mapping_to_class function. - """ - # Add a single mapping and validate that all it did was add a class - # mapping and that the mapping has the expected values. - key = ('foo', 'Foo') - class Bar: - pass - value = (Bar.__module__, 'Bar') - self.registry.add_mapping_to_class(key[0], key[1], Bar) - self._validate_exactly_one_class_map(key, value) - - # Overwrite with a new mapping and validate the state is what we - # expect. - class Baz: - pass - value = (Baz.__module__, 'Baz') - self.registry.add_mapping_to_class(key[0], key[1], Baz) - self._validate_exactly_one_class_map(key, value) - - - def test_add_mappings(self): - """ Validates the behavior of the add_mappings function. - """ - # Add a single mapping and validate that all it did was add a class - # mapping and that the mapping has the expected values - key = ('foo', 'Foo') - value = ('bar', key[1]) - names = [key[1]] - self.registry.add_mappings(key[0], value[0], names) - self._validate_exactly_one_class_map(key, value) - - # Add multiple mappings and validate that the registry has the expected - # values. - key = ('foo', 'Foo') - value = ('bar', 'Bar') - names = ['Foo', 'Bar', 'Baz', 'Enthought'] - self.registry.add_mappings(key[0], value[0], names) - self.assertEqual(len(names), len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - items = [] - for n in names: - items.append( ((key[0], n), (value[0], n)) ) - self._validate_class_map_contents(items) - - - def test_add_state_function(self): - """ Validates the behavior of the add_state_function function. - """ - # Add a single function and validate that all it did was add a state - # mapping and that the mapping has the expected values. - key = ('foo', 'Foo', 1) - def fn(): - pass - self.registry.add_state_function(key[0], key[1], key[2], fn) - self._validate_exactly_one_state_function(key, [fn]) - - # Add an additional function for the same state and validate the state - # is what we expect. - def fn2(): - pass - self.registry.add_state_function(key[0], key[1], key[2], fn2) - self._validate_exactly_one_state_function(key, [fn, fn2]) - - # Add a state function for another version of the same class and - # validate that all the values are as expected. - key2 = ('foo', 'Foo', 2) - self.registry.add_state_function(key2[0], key2[1], key2[2], fn2) - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(2, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self._validate_state_function_contents( - [(key, [fn, fn2]), (key2, [fn2])], - {(key[0], key[1]): 3} - ) - - - def test_add_state_function_for_class(self): - """ Validates the behavior of the add_state_function_for_class function. - """ - # Add a single function and validate that all it did was add a state - # mapping and that the mapping has the expected values. - class Bar: - pass - key = (Bar.__module__, 'Bar', 1) - def fn(): - pass - self.registry.add_state_function_for_class(Bar, key[2], fn) - self._validate_exactly_one_state_function(key, [fn]) - - # Add an additional function for the same state and validate the state - # is what we expect. - def fn2(): - pass - self.registry.add_state_function_for_class(Bar, key[2], fn2) - self._validate_exactly_one_state_function(key, [fn, fn2]) - - # Add a state function for another class and validate that all the - # values are as expected. - class Baz: - pass - key2 = (Baz.__module__, 'Baz', 2) - self.registry.add_state_function_for_class(Baz, key2[2], fn2) - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(2, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(2, len(self.registry._state_function_classes)) - self._validate_state_function_contents( - [(key, [fn, fn2]), (key2, [fn2])], - {(key[0], key[1]): 2, (key2[0], key2[1]): 1} - ) - - - def test_merge_updater(self): - """ Validates the behavior of the merge_updater function. - """ - # Merge in one update and validate the state of the registry is as - # expected. - def fn1(): - pass - updater = sweet_pickle.Updater( - class_map = { - ('foo', 'Foo'): ('foo.bar', 'Foo'), - }, - state_functions = { - ('foo', 'Foo', 1): [fn1], - }, - version_attribute_map = { - ('foo', 'Foo'): 'version', - }, - ) - self.registry.merge_updater(updater) - self.assertEqual(1, len(self.registry.class_map)) - self.assertEqual(1, len(self.registry.state_functions)) - self.assertEqual(1, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self._validate_class_map_contents(list(updater.class_map.items())) - counts = {('foo', 'Foo'): 1} - self._validate_state_function_contents(list(updater.state_functions.items()), - counts) - - # Merge in a second updater and validate the state of the registry is - # as expected. - def fn2(): - pass - updater2 = sweet_pickle.Updater( - class_map = { - ('foo.bar', 'Foo'): ('bar', 'Bar'), - ('bar', 'Bar'): ('foo.bar.baz', 'Baz'), - }, - state_functions = { - ('foo', 'Foo', 1): [fn2], - ('foo', 'Foo', 2): [fn2], - }, - version_attribute_map = { - ('foo.bar', 'Foo'): '_version', - }, - ) - self.registry.merge_updater(updater2) - self.assertEqual(3, len(self.registry.class_map)) - self.assertEqual(2, len(self.registry.state_functions)) - self.assertEqual(2, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self._validate_class_map_contents( - list(updater.class_map.items()) + list(updater2.class_map.items()) - ) - counts = {('foo', 'Foo'): 3} - self._validate_state_function_contents( - [ (('foo', 'Foo', 1), [fn1, fn2]), (('foo', 'Foo', 2), [fn2]) ], - counts) - - - def test_registry_starts_empty(self): - """ Validates that the registry is starting empty for each test. - """ - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - - - ### protected interface ################################################## - - def _validate_class_map_contents(self, items): - """ Validates that the registry's class_map contains the specified - items. - """ - for key, value in items: - self.assertEqual(True, key in self.registry.class_map, - 'Key ' + str(key) + ' not in class_map') - self.assertEqual(value, self.registry.class_map[key], - str(value) + ' != ' + str(self.registry.class_map[key]) + \ - ' for key ' + str(key)) - self.assertEqual(True, - self.registry.has_class_mapping(key[0], key[1]), - 'Registry reports no class mapping for key ' + str(key)) - - - def _validate_exactly_one_class_map(self, key, value): - """ Validates that the registry has exactly one class_map entry - with the specified key and value. - """ - self.assertEqual(1, len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - self._validate_class_map_contents([(key, value)]) - - - def _validate_exactly_one_state_function(self, key, value): - """ Validates that the registry has exactly one state_function entry - with the specified key and value. - """ - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(1, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self.assertEqual(key, list(self.registry.state_functions.keys())[0]) - self.assertEqual(value, self.registry.state_functions[key]) - classes_key = (key[0], key[1]) - self.assertEqual(classes_key, - list(self.registry._state_function_classes.keys())[0]) - self.assertEqual(len(value), - self.registry._state_function_classes[classes_key]) - - - def _validate_state_function_contents(self, items, counts): - """ Validates that the registry's state functions contains the - specified items and the class count matches the specified count. - """ - for key, value in items: - self.assertEqual(True, key in self.registry.state_functions, - 'Key ' + str(key) + ' not in state functions') - self.assertEqual(value, self.registry.state_functions[key], - str(value) + ' != ' + \ - str(self.registry.state_functions[key]) + ' for key ' + \ - str(key)) - self.assertEqual(True, self.registry.has_state_function( - key[0], key[1]), - 'Registry reports no state function for key ' + str(key)) - - classes_key = (key[0], key[1]) - count = counts[classes_key] - self.assertEqual(count, - self.registry._state_function_classes[classes_key]) diff --git a/apptools/sweet_pickle/updater.py b/apptools/sweet_pickle/updater.py deleted file mode 100644 index 702036470..000000000 --- a/apptools/sweet_pickle/updater.py +++ /dev/null @@ -1,300 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2005, 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# Author: Duncan Child -# -#----------------------------------------------------------------------------- - -""" A record of refactorings to be performed during unpickling of objects. -""" - -# Standard library imports -import logging - -# Enthought library imports -from traits.api import Dict, HasPrivateTraits, Int, List, Tuple, Str - - -logger = logging.getLogger(__name__) - - -############################################################################## -# class 'Updater' -############################################################################## - -class Updater(HasPrivateTraits): - """ A record of refactorings to be performed during unpickling of objects. - """ - - ########################################################################## - # Traits - ########################################################################## - - ### public 'Updater' interface ########################################### - - # Mappings from a pickled class to a class it should be unpickled as. - # - # The keys are a tuple of the source class's module and class names in - # that order. The values are the target class's module and class names - # in that order. - class_map = Dict(Tuple(Str, Str), Tuple(Str, Str)) - - # State functions that should be called to convert state from one version - # of a class to another. - # - # The keys are a tuple of the class's module name, class name, and version - # in that order. The values are a list of functions to be called during - # unpickling to do the state conversion. Note that the version in the - # key represents the version that the function converts *TO*. - state_functions = Dict(Tuple(Str, Str, Int), List) - - # Our record of the attribute that records the version number for a - # specific class. If no record is found for a given class, then the - # default value is used instead -- see '_default_version_attribute'. - # - # The key is a tuple of the class's module name and class name in that - # order. The value is the name of the version attribute. - version_attribute_map = Dict(Tuple(Str, Str), Str) - - - ### protected 'Updater' interface ######################################## - - # The default name of the attribute that declares the version of a class - # or class instance. - _default_version_attribute = '_enthought_pickle_version' - - # A record of which classes we have state functions for. - # - # The keys are a tuple of the class's module name and class name in that - # order. The values are reference counts. - _state_function_classes = Dict(Tuple(Str, Str), Int) - - - ########################################################################## - # 'Updater' interface - ########################################################################## - - ### public interface ##################################################### - - def add_mapping(self, source_module, source_name, target_module, - target_name): - """ Adds a mapping from the class with the source name in the source - module to the class with the target name in the target module. - """ - self.class_map[(source_module, source_name)] = (target_module, - target_name) - - - def add_mapping_to_class(self, source_module, source_name, target_class): - """ Convenience method to add a mapping, from the class with the - source name in the source module to the target class. - """ - self.add_mapping(source_module, source_name, target_class.__module__, - target_class.__name__) - - - def add_mappings(self, source_module, target_module, class_names): - """ Adds mappings, from the specified source module to the specified - target module, for each of the class names in the specified - list. - """ - for name in class_names: - self.add_mapping(source_module, name, target_module, name) - - - def add_state_function(self, module, name, target_version, function): - """ Adds the specified function as a state function to be called to - convert an instance of the class with the specified name within - the specified module *TO* the specified version. - - Note that the framework handles calling of state functions to make - the smallest version jumps possible. - """ - key = (module, name, target_version) - list = self.state_functions.setdefault(key, []) - list = list[:] # Copy necessary because traits only recognizes list - # changes by list instance - not its contents. - list.append(function) - self.state_functions[key] = list - - - def add_state_function_for_class(self, klass, target_version, function): - """ Convenience method to add the specified function as a state - function to be called to convert an instance of the specified - class *TO* the specified version. - """ - self.add_state_function(klass.__module__, klass.__name__, - target_version, function) - - - def declare_version_attribute(self, module, name, attribute_name): - """ Adds the specified attribute name as the version attribute for the - class within the specified module with the specified name. - """ - self.version_attribute_map[(module, name)] = attribute_name - - - def declare_version_attribute_for_class(self, klass, attribute_name): - """ Covenience method to add the specified attribute name as the - version attribute for the specified class. - """ - self.declare_version_attribute(klass.__module__, klass.__name__, - attribute_name) - - - def get_version_attribute(self, module, name): - """ Returns the name of the version attribute for the class of the - specified name within the specified module. - """ - return self.version_attribute_map.get( (module, name), - self._default_version_attribute) - - - def has_class_mapping(self, module, name): - """ Returns True if this updater contains a class mapping for - the class identified by the specified module and class name. - """ - return (module, name) in self.class_map - - - def has_state_function(self, module, name): - """ Returns True if this updater contains any state functions for - the class identified by the specified module and class name. - """ - return (module, name) in self._state_function_classes - - - def merge_updater(self, updater): - """ Merges the mappings and state functions from the specified updater - into this updater. - """ - self.class_map.update(updater.class_map) - self.version_attribute_map.update(updater.version_attribute_map) - - # The state functions dictionary requires special processing because - # each value is a list and we don't just want to replace the existing - # list with only the new content. - for key, value in updater.state_functions.items(): - if isinstance(value, list) and len(value) > 0: - funcs = self.state_functions.setdefault(key, []) - funcs = funcs[:] # Copy necessary because traits only recognizes - # funcs changes by funcs instance - not its - # contents. - funcs.extend(value) - self.state_functions[key] = funcs - - - ### trait handlers ####################################################### - - def _class_map_changed(self, old, new): - logger.debug('Detected class_map change from [%s] to [%s] in [%s]', - old, new, self) - - - def _class_map_items_changed(self, event): - for o in event.removed: - logger.debug('Detected [%s] removed from class_map in [%s]', o, - self) - for k, v in event.changed.items(): - logger.debug('Detected [%s] changed from [%s] to [%s] in ' + \ - 'class_map in [%s]', k, v, self.class_map[k], self) - for k, v in event.added.items(): - logger.debug('Detected mapping from [%s] to [%s] added to ' + \ - 'class_map in [%s]', k, v, self) - - - def _state_functions_changed(self, old, new): - logger.debug('Detected state_functions changed from [%s] to [%s] ' + \ - 'in [%s]', old, new, self) - - # Update our record of which classes we have state functions for. - # All of our old state functions are gone so we simply need to rescan - # the new functions. - self._state_function_classes.clear() - for key, value in new.items(): - module, name, version = key - klass_key = (module, name) - count = self._state_function_classes.setdefault(klass_key, 0) - self._state_function_classes[klass_key] = count + len(value) - - - def _state_functions_items_changed(self, event): - # Decrement our reference counts for the classes we no longer - # have state functions for. If the reference count reaches zero, - # remove the record completely. - for k, v in event.removed.items(): - logger.debug('Detected [%s] removed from state_functions in [%s]', - k, self) - - # Determine the new reference count of state functions for the - # class who the removed item was for. - module, name, version = k - key = (module, name) - count = self._state_function_classes[key] - len(v) - - # Store the new reference count. Delete the entry if it is zero. - if count < 0: - logger.warn('Unexpectedly reached negative reference count ' + - 'value of [%s] for [%s]', count, key) - del self._state_function_classes[key] - elif count == 0: - del self._state_function_classes[key] - else: - self._state_function_classes[key] = count - - # Update our reference counts for changes to the list of functions - # for a specific class and version. The 'changed' dictionary's values - # are the old values. - for k, v in event.changed.items(): - value = self.state_functions[k] - logger.debug('Detected [%s] changed in state_functions from ' + \ - '[%s] to [%s] in [%s]', k, v, value, self) - - # Determine the new reference count as a result of the change. - module, name, version = k - key = (module, name) - count = self._state_function_classes[key] - len(v) + len(value) - - # Store the new reference count. Delete the entry if it is zero. - if count < 0: - logger.warn('Unexpectedly reached negative reference count ' + - 'value of [%s] for [%s]', count, key) - del self._state_function_classes[key] - elif count == 0: - del self._state_function_classes[key] - else: - self._state_function_classes[key] = count - - - # Update our reference counts for newly registered state functions. - for k, v in event.added.items(): - logger.debug('Detected mapping of [%s] to [%s] added to ' + \ - 'state_functions in [%s]', k, v, self) - - # Determine the new reference count as a result of the change. - module, name, version = k - key = (module, name) - count = self._state_function_classes.setdefault(key, 0) + len(v) - - # Store the new reference count - self._state_function_classes[key] = count - - - def _version_attribute_map_changed(self, old, new): - logger.debug('Detected version_attribute_map change from [%s] ' + \ - 'to [%s] in [%s]', old, new, self) - - - def _version_attribute_map_items_changed(self, event): - for o in event.removed: - logger.debug('Detected [%s] removed from version_attribute_map ' + \ - 'in [%s]', o, self) - for o in event.changed: - logger.debug('Detected [%s] changed in version_attribute_map ' + \ - 'in [%s]', o, self) - for o in event.added: - logger.debug('Detected [%s] added to version_attribute_map in ' + \ - '[%s]', o, self) diff --git a/apptools/sweet_pickle/versioned_unpickler.py b/apptools/sweet_pickle/versioned_unpickler.py deleted file mode 100644 index a96e37a9b..000000000 --- a/apptools/sweet_pickle/versioned_unpickler.py +++ /dev/null @@ -1,507 +0,0 @@ -#----------------------------------------------------------------------------- -# -# Copyright (c) 2005-2008 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# Author: Duncan Child -# -#----------------------------------------------------------------------------- -# The code for two-stage unpickling support has been taken from a PEP draft -# prepared by Dave Peterson and Prabhu Ramachandran. - -""" An unpickler that is tolerant of class refactorings, and implements a -two-stage pickling process to make it possible to unpickle complicated Python -object hierarchies where the unserialized state of an object depends on the -state of other objects in the same pickle. -""" - -# Standard library imports. -import sys -import logging -from os import path -from types import GeneratorType - -if sys.version_info[0] >= 3: - from pickle import _Unpickler as Unpickler -else: - from pickle import Unpickler - -from pickle import UnpicklingError, BUILD - -# Enthought library imports -from traits.api import HasTraits, Instance - - -# Setup a logger for this module -logger = logging.getLogger(__name__) - - -############################################################################## -# constants -############################################################################## - -# The name we backup the original setstate method to. -_BACKUP_NAME = '__enthought_sweet_pickle_original_setstate__' - -# The name of the setstate method we hook -_SETSTATE_NAME = '__setstate__' - -# The name we store our unpickling data under. -_UNPICKLER_DATA = '__enthought_sweet_pickle_unpickler__' - - -############################################################################## -# function '__replacement_setstate__' -############################################################################## - -def __replacement_setstate__(self, state): - """ Called to enable an unpickler to modify the state of this instance. - """ - # Retrieve the unpickling information and use it to let the unpickler - # modify our state. - unpickler, module, name = getattr(self, _UNPICKLER_DATA) - state = unpickler.modify_state(self, state, module, name) - - # If we were given a state, apply it to this instance now. - if state is not None: - - # Save our state - logger.debug('Final state: %s', state) - self.__dict__.update(state) - - -############################################################################## -# function 'load_build_with_meta_data' -############################################################################## - -def load_build_with_meta_data(self): - """ Called prior to the actual load_build() unpickling method which primes - the state dictionary with meta-data. - """ - - # Access the state object and check if it is a dictionary (state may also be - # a tuple, which is used for other unpickling build operations). Proceed to - # the standard load_build() if the state obj is not a dict. - state = self.stack[-1] - if type(state) == dict: - - # If a file object is used, reference the file name - if hasattr(self._file, 'name'): - pickle_file_name = path.abspath(self._file.name) - else : - pickle_file_name = "" - - # Add any meta-data needed by __setstate__() methods here... - state['_pickle_file_name'] = pickle_file_name - - # Call the standard load_build() method - return self.load_build() - - -############################################################################## -# class 'NewUnpickler' -############################################################################## -class NewUnpickler(Unpickler): - """ An unpickler that implements a two-stage pickling process to make it - possible to unpickle complicated Python object hierarchies where the - unserialized state of an object depends on the state of other objects in - the same pickle. - """ - - def load(self, max_pass=-1): - """Read a pickled object representation from the open file. - - Return the reconstituted object hierarchy specified in the file. - """ - # List of objects to be unpickled. - self.objects = [] - - # We overload the load_build method. - dispatch = self.dispatch - dispatch[BUILD[0]] = NewUnpickler.load_build - - # call the super class' method. - ret = Unpickler.load(self) - self.initialize(max_pass) - self.objects = [] - - # Reset the Unpickler's dispatch table. - dispatch[BUILD[0]] = Unpickler.load_build - return ret - - def initialize(self, max_pass): - # List of (object, generator) tuples that initialize objects. - generators = [] - - # Execute object's initialize to setup the generators. - for obj in self.objects: - if hasattr(obj, '__initialize__') and \ - callable(obj.__initialize__): - ret = obj.__initialize__() - if isinstance(ret, GeneratorType): - generators.append((obj, ret)) - elif ret is not None: - raise UnpicklingError('Unexpected return value from ' - '__initialize__. %s returned %s' % (obj, ret)) - - # Ensure a maximum number of passes - if max_pass < 0: - max_pass = len(generators) - - # Now run the generators. - count = 0 - while len(generators) > 0: - count += 1 - if count > max_pass: - not_done = [x[0] for x in generators] - msg = """Reached maximum pass count %s. You may have - a deadlock! The following objects are - uninitialized: %s""" % (max_pass, not_done) - raise UnpicklingError(msg) - for o, g in generators[:]: - try: - next(g) - except StopIteration: - generators.remove((o, g)) - - # Make this a class method since dispatch is a class variable. - # Otherwise, supposing the initial sweet_pickle.load call (which would - # have overloaded the load_build method) makes a pickle.load call at some - # point, we would have the dispatch still pointing to - # NewPickler.load_build whereas the object being passed in will be an - # Unpickler instance, causing a TypeError. - @classmethod - def load_build(cls, obj): - # Just save the instance in the list of objects. - if isinstance(obj, NewUnpickler): - obj.objects.append(obj.stack[-2]) - Unpickler.load_build(obj) - - -############################################################################## -# class 'VersionedUnpickler' -############################################################################## - -class VersionedUnpickler(NewUnpickler, HasTraits): - """ An unpickler that is tolerant of class refactorings. - - This class reads in a pickled file and applies the transforms - specified in its updater to generate a new hierarchy of objects - which are at the current version of the classes they are instances - of. - - Note that the creation of an updater is kept out of this class to - ensure that the class can be reused in different situations. - However, if no updater is provided during construction, then the - global registry updater will be used. - """ - - ########################################################################## - # Traits - ########################################################################## - - ### public 'VersionedUnpickler' interface ################################ - - # The updater used to modify the objects being unpickled. - updater = Instance('apptools.sweet_pickle.updater.Updater') - - - ########################################################################## - # 'object' interface - ########################################################################## - - ### operator methods ##################################################### - - def __init__(self, file, **kws): - super(VersionedUnpickler, self).__init__(file) - - self._file = file - if self.updater is None: - from .global_registry import get_global_registry - self.updater = get_global_registry() - logger.debug('VersionedUnpickler [%s] using Updater [%s]', self, - self.updater) - - # Update the BUILD instruction to use an overridden load_build method - # NOTE: this is being disabled since, on some platforms, the object - # is replaced with a regular Unpickler instance, creating a traceback: - # AttributeError: Unpickler instance has no attribute '_file' - # ...not sure how this happens since only a VersionedUnpickler has - # the BUILD instruction replaced with one that uses _file, and it - # should have _file defined. - #self.dispatch[BUILD[0]] = load_build_with_meta_data - - - ########################################################################## - # 'Unpickler' interface - ########################################################################## - - ### public interface ##################################################### - - def find_class(self, module, name): - """ Returns the class definition for the named class within the - specified module. - - Overridden here to: - - - Allow updaters to redirect to a different class, possibly - within a different module. - - Ensure that any setstate hooks for the class are called - when the instance of this class is unpickled. - """ - # Remove any extraneous characters that an Unpickler might handle - # but a user wouldn't have included in their mapping definitions. - module = module.strip() - name = name.strip() - - # Attempt to find the class, this may cause a new mapping for that - # very class to be introduced. That's why we ignore the result. - try: - klass = super(VersionedUnpickler, self).find_class(module, name) - except: - pass - - # Determine the target class that the requested class should be - # mapped to according to our updater. The target class is the one - # at the end of any chain of mappings. - original_module, original_name = module, name - if self.updater is not None and \ - self.updater.has_class_mapping(module, name): - module, name = self._get_target_class(module, name) - if module != original_module or name != original_name: - logger.debug('Unpickling [%s.%s] as [%s.%s]', original_module, - original_name, module, name) - - # Retrieve the target class definition - try: - klass = super(VersionedUnpickler, self).find_class(module, name) - except Exception as e: - from apptools.sweet_pickle import UnpicklingError - logger.debug('Traceback when finding class [%s.%s]:' \ - % (module, name), exc_info=True) - raise UnpicklingError('Unable to load class [%s.%s]. ' - 'Original exception was, "%s". map:%s' % ( - module, name, str(e), self.updater.class_map)) - - # Make sure we run the updater's state functions if any are declared - # for the target class. - if self.updater is not None \ - and self._has_state_function(original_module, original_name): - self._add_unpickler(klass, original_module, original_name) - - return klass - - - ########################################################################## - # 'VersionedUnpickler' interface - ########################################################################## - - ### public interface ##################################################### - - def modify_state(self, obj, state, module, name): - """ Called to update the specified state dictionary, which represents - the class of the specified name within the specified module, to - complete the unpickling of the specified object. - """ - # Remove our setstate hook and associated data to ensure that - # instances unpickled through some other framework don't call us. - # IMPORTANT: Do this first to minimize the time this hook is in place! - self._remove_unpickler(obj.__class__) - - # Determine what class and version we're starting from and going to. - # If there is no version information, then assume version 0. (0 is - # like an unversioned version.) - source_key = self.updater.get_version_attribute(module, name) - source_version = state.get(source_key, 0) - target_key = self.updater.get_version_attribute( - obj.__class__.__module__, obj.__class__.__name__) - target_version = getattr(obj, target_key, 0) - - # Iterate through all the updates to the state by going one version - # at a time. Note that we assume there is exactly one path from our - # starting class and version to our ending class and version. As a - # result, we assume we update a given class to its latest version - # before looking for any class mappings. Note that the version in the - # updater is the version to convert *TO*. - version = source_version - next_version = version + 1 - while True: - - # Iterate through all version updates for the current class. - key = self.updater.get_version_attribute(module, name) - while (module, name, next_version) in self.updater.state_functions: - functions = self.updater.state_functions[(module, name, - next_version)] - for f in functions: - logger.debug('Modifying state from [%s.%s (v.%s)] to ' + \ - '[%s.%s (v.%s)] using function %s', module, name, - version, module, name, next_version, f) - state = f(state) - - # Avoid infinite loops due to versions not changing. - new_version = state.get(key, version) - if new_version == version: - new_version = version + 1 - version = new_version - next_version = version + 1 - - # If there is one, move to the next class in the chain. (We - # explicitly keep the version number the same.) - if self.updater.has_class_mapping(module, name): - original_module, original_name = module, name - module, name = self.updater.class_map[(module, name)] - logger.debug('Modifying state from [%s.%s (v.%s)] to ' + \ - '[%s.%s (v.%s)]', original_module, original_name, version, - module, name, version) - else: - break - - # If one exists, call the final class's setstate method. According to - # standard pickling protocol, this method will apply the state to the - # instance so our state becomes None so that we don't try to apply our - # unfinished state to the object. - fn = getattr(obj, _SETSTATE_NAME, None) - if fn is not None: - fn(state) - result = None - version = getattr(obj, target_key) - else: - result = state - - # Something is wrong if we aren't at our target class and version! - if module != obj.__class__.__module__ \ - or name != obj.__class__.__name__ \ - or version != target_version: - from apptools.sweet_pickle import UnpicklingError - raise UnpicklingError('Unexpected state! Got ' + \ - '[%s.%s (v.%s)] expected [%s.%s (v.%s)]' % (module, name, - version, obj.__class__.__module__, obj.__class__.__name__, - target_version)) - - return result - - - ### protected interface ################################################## - - def _add_unpickler(self, klass, module, name): - """ Modifies the specified class so that our 'modify_state' method - is called when its next instance is unpickled. - """ - logger.debug('Adding unpickler hook to [%s]', klass) - - # Replace the existing setstate method with ours. - self._backup_setstate(klass) - m = __replacement_setstate__.__get__(None, klass) - setattr(klass, _SETSTATE_NAME, m) - - # Add the information necessary to allow this unpickler to run - setattr(klass, _UNPICKLER_DATA, (self, module, name)) - - - def _backup_setstate(self, klass): - """ Backs up the specified class's setstate method. - """ - # We only need to back it up if it actually exists. - method = getattr(klass, _SETSTATE_NAME, None) - if method is not None: - logger.debug('Backing up method [%s] to [%s] on [%s]', - _SETSTATE_NAME, _BACKUP_NAME, klass) - m = method.__get__(None, klass) - setattr(klass, _BACKUP_NAME, m) - - - def _get_target_class(self, module, name): - """ Returns the class info that the class, within the specified module - and with the specified name, should be instantiated as according to - our associated updater. - - This is done in a manner that allows for chaining of class mappings - but is tolerant of the fact that a mapping away from an - intermediate class may not be registered until an attempt is made - to load that class. - """ - # Keep a record of the original class asked for. - original_module, original_name = module, name - - # Iterate through any mappings in a manner that allows us to detect any - # infinite loops. - visited = [] - while self.updater.has_class_mapping(module, name): - if (module, name) in visited: - from apptools.sweet_pickle import UnpicklingError - raise UnpicklingError('Detected infinite loop in class ' + \ - 'mapping from [%s.%s] to [%s.%s] within Updater [%s]' % \ - (original_module, original_name, module, name, - self.updater)) - visited.append( (module, name) ) - - # Get the mapping for the current class and try loading the class - # to ensure any mappings away from it are registered. - module, name = self.updater.class_map[(module, name)] - try: - super(VersionedUnpickler, self).find_class(module, name) - except: - logger.exception("_get_target_class can't find: %s" % (module, name)) - pass - - return module, name - - - def _has_state_function(self, module, name): - """ Returns True if the updater contains any state functions that could - be called by unpickling an instance of the class identified by the - specified module and name. - - Note: If we had a version number we could tell for sure, but we - don't have one so we'll have to settle for 'could' be called. - """ - result = False - - # Iterate through all the class mappings the requested class would - # go through. If any of them have a state function, then we've - # determined our answer and can stop searching. - # - # Note we don't need to check for infinite loops because we're only - # ever called after '_get_target_class' which detects the infinite - # loops. - while not result: - result = self.updater.has_state_function(module, name) - if not result: - if self.updater.has_class_mapping(module, name): - module, name = self.updater.class_map[(module, name)] - else: - break - - return result - - - def _remove_unpickler(self, klass): - """ Restores the specified class to its unmodified state. Meaning - we won't get called when its next instance is unpickled. - """ - logger.debug('Removing unpickler hook from [%s]', klass) - - # Restore the backed up setstate method - self._restore_setstate(klass) - - # Remove the unpickling data attached to the class. This ensures we - # don't pollute the 'real' attributes of the class. - delattr(klass, _UNPICKLER_DATA) - - - def _restore_setstate(self, klass): - """ Restores the original setstate method back to its rightful place. - """ - # We only need to restore if the backup actually exists. - method = getattr(klass, _BACKUP_NAME, None) - if method is not None: - logger.debug('Restoring method [%s] to [%s] on [%s]', - _BACKUP_NAME, _SETSTATE_NAME, klass) - delattr(klass, _BACKUP_NAME) - m = method.__get__(None, klass) - setattr(klass, _SETSTATE_NAME, m) - - # Otherwise, we simply remove our setstate. - else: - delattr(klass, _SETSTATE_NAME) From 71bba6eb92b903e5661ae2d39c74f0fbf1a1e384 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Wed, 11 Nov 2020 10:47:04 -0600 Subject: [PATCH 02/14] adding news fragment --- docs/releases/upcoming/199.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/releases/upcoming/199.removal.rst diff --git a/docs/releases/upcoming/199.removal.rst b/docs/releases/upcoming/199.removal.rst new file mode 100644 index 000000000..c22d99e8c --- /dev/null +++ b/docs/releases/upcoming/199.removal.rst @@ -0,0 +1 @@ +remove the ``apptools.sweet_pickle`` subpackage. Note that users of sweet_pickle can transition to using ``apptools.persistance`` and pickle from the python standard library (see changes made in this PR to ``apptools.naming`` for more info) (#199) \ No newline at end of file From d8f436a5bbc73fbf0afc4a918f8e09521fc38573 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Fri, 20 Nov 2020 15:38:32 -0600 Subject: [PATCH 03/14] remove @classmethod on load_build --- apptools/persistence/versioned_unpickler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apptools/persistence/versioned_unpickler.py b/apptools/persistence/versioned_unpickler.py index 1980ab267..21d16ba94 100644 --- a/apptools/persistence/versioned_unpickler.py +++ b/apptools/persistence/versioned_unpickler.py @@ -83,7 +83,6 @@ def initialize(self, max_pass): # point, we would have the dispatch still pointing to # NewPickler.load_build whereas the object being passed in will be an # Unpickler instance, causing a TypeError. - @classmethod def load_build(cls, obj): # Just save the instance in the list of objects. if isinstance(obj, NewUnpickler): From 6de956b3c683b01739d13f56d3b4ab70649bfb4d Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Fri, 20 Nov 2020 16:02:12 -0600 Subject: [PATCH 04/14] finish merge --- apptools/naming/object_serializer.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/apptools/naming/object_serializer.py b/apptools/naming/object_serializer.py index 626c2e0d4..204959eb5 100644 --- a/apptools/naming/object_serializer.py +++ b/apptools/naming/object_serializer.py @@ -55,13 +55,7 @@ def load(self, path): f = open(path, "rb") try: try: -<<<<<<< HEAD obj = VersionedUnpickler(f).load() -======= - obj = sweet_pickle.load(f) - # obj = cPickle.load(f) - # obj = pickle.load(f) ->>>>>>> master except Exception as ex: print_exc() logger.exception( @@ -91,13 +85,7 @@ def save(self, path, obj): # Pickle the object. f = open(actual_path, "wb") try: -<<<<<<< HEAD pickle.dump(obj, f, 1) -======= - sweet_pickle.dump(obj, f, 1) - # cPickle.dump(obj, f, 1) - # pickle.dump(obj, f, 1) ->>>>>>> master except Exception as ex: logger.exception( "Failed to pickle into file: %s, %s, object:%s" From 9e4f02902e9e9661d57e83f8ec750a5298afd864 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Mon, 23 Nov 2020 13:11:39 -0700 Subject: [PATCH 05/14] add roundtrip test for object serializer --- .../naming/tests/test_object_serializer.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 apptools/naming/tests/test_object_serializer.py diff --git a/apptools/naming/tests/test_object_serializer.py b/apptools/naming/tests/test_object_serializer.py new file mode 100644 index 000000000..d28baa797 --- /dev/null +++ b/apptools/naming/tests/test_object_serializer.py @@ -0,0 +1,52 @@ +# (C) Copyright 2005-2020 Enthought, Inc., Austin, TX +# All rights reserved. +# +# This software is provided without warranty under the terms of the BSD +# license included in LICENSE.txt and may be redistributed only under +# the conditions described in the aforementioned license. The license +# is also available online at http://www.enthought.com/licenses/BSD.txt +# +# Thanks for using Enthought open source! +import os +import shutil +import tempfile +import unittest + +from traits.api import cached_property, HasTraits, Property, Str, Event + +from apptools.naming.api import ObjectSerializer + + +class FooWithTraits(HasTraits): + """Dummy HasTraits class for testing ObjectSerizalizer.""" + + full_name = Str() + + last_name = Property(depends_on="full_name") + + event = Event() + + @cached_property + def _get_last_name(self): + return self.full_name.split(" ")[-1] + +class TestObjectSerializer(unittest.TestCase): + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmpdir) + self.tmp_file = os.path.join(self.tmpdir, "tmp.pickle") + + def test_save_load_roundtrip(self): + # Test HasTraits objects can be serialized and deserialized as expected + obj = FooWithTraits(full_name="John Doe") + + serializer = ObjectSerializer() + serializer.save(self.tmp_file, obj) + + self.assertTrue(serializer.can_load(self.tmp_file)) + deserialized = serializer.load(self.tmp_file) + + self.assertIsInstance(deserialized, FooWithTraits) + self.assertEqual(deserialized.full_name, "John Doe") + self.assertEqual(deserialized.last_name, "Doe") From 084d27d114927cd840c2ebeeb653c273b6f07645 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Mon, 23 Nov 2020 14:26:39 -0700 Subject: [PATCH 06/14] rewrite test_two_stage_unpickler to use persistence instead of sweet pickle and remove test_global registry --- .../tests/test_two_stage_unpickler.py | 46 ++++-------- .../tests/test_global_registry.py | 70 ------------------- 2 files changed, 15 insertions(+), 101 deletions(-) rename apptools/{sweet_pickle => persistence}/tests/test_two_stage_unpickler.py (79%) delete mode 100644 apptools/sweet_pickle/tests/test_global_registry.py diff --git a/apptools/sweet_pickle/tests/test_two_stage_unpickler.py b/apptools/persistence/tests/test_two_stage_unpickler.py similarity index 79% rename from apptools/sweet_pickle/tests/test_two_stage_unpickler.py rename to apptools/persistence/tests/test_two_stage_unpickler.py index c52e5b10c..0231dac85 100644 --- a/apptools/sweet_pickle/tests/test_two_stage_unpickler.py +++ b/apptools/persistence/tests/test_two_stage_unpickler.py @@ -7,14 +7,17 @@ # Prabhu Ramachandran # ----------------------------------------------------------------------------- -# Test cases. -from __future__ import print_function +""" This was previously a test for the now deleted apptools.sweet_pickle +sub package. It is included here to showcase how apptools.persistance can be +used to replace sweet_pickle functionality. +""" +import io import random import pickle import unittest -import apptools.sweet_pickle as sweet_pickle +from apptools.persistence.versioned_unpickler import VersionedUnpickler ######################################## @@ -66,8 +69,6 @@ def __initialize__(self): class GenericTestCase(unittest.TestCase): def test_generic(self): - print("\nRunning generic test...") - a = A() b = B() a.x = random.randint(1, 100) @@ -79,20 +80,16 @@ def test_generic(self): s = pickle.dumps(a) new_a = pickle.loads(s) try: - print("\ta.x: %s" % new_a.x) - print("\ta.b_ref.y: %s" % new_a.b_ref.y) - except Exception as msg: - print("\t%s" % "Expected Error".center(75, "*")) - print("\t%s" % msg) - print("\t%s" % ("*" * 75)) + new_a.x + new_a.b_ref.y + except Exception: + pass # This will work! s = pickle.dumps(a) - new_a = sweet_pickle.loads(s) + new_a = VersionedUnpickler(io.BytesIO(s)).load() assert new_a.x == new_a.b_ref.y == value - print("Generic test succesfull.\n\n") - ######################################## # Usecase2: Toy Application @@ -154,14 +151,11 @@ def __init__(self): self.finder = StringFinder(self.reader, "e") def get(self): - print("\t%s" % self.finder.data) - print("\t%s" % self.reader.data) + pass class ToyAppTestCase(unittest.TestCase): def test_toy_app(self): - print("\nRunning toy app test...") - a = Application() a.finder.find() a.get() @@ -170,19 +164,9 @@ def test_toy_app(self): # Won't work. try: b.get() - except Exception as msg: - print("\t%s" % "Expected Error".center(75, "*")) - print("\t%s" % msg) - print("\t%s" % ("*" * 75)) + except Exception: + pass # Works fine. - c = sweet_pickle.loads(s) + c = VersionedUnpickler(io.BytesIO(s)).load() c.get() - - print("Toy app test succesfull.\n\n") - - -if __name__ == "__main__": - test_generic() - test_toy_app() - print("ALL TESTS SUCCESFULL\n") diff --git a/apptools/sweet_pickle/tests/test_global_registry.py b/apptools/sweet_pickle/tests/test_global_registry.py deleted file mode 100644 index 10607e649..000000000 --- a/apptools/sweet_pickle/tests/test_global_registry.py +++ /dev/null @@ -1,70 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -# ----------------------------------------------------------------------------- - -""" Tests the global registry functionality of the sweet_pickle framework. -""" - -# Standard library imports. -import unittest - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry - - -############################################################################## -# class 'GlobalRegistryTestCase' -############################################################################## - - -class GlobalRegistryTestCase(unittest.TestCase): - """Tests the global registry functionality of the sweet_pickle framework.""" - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - ########################################################################## - # 'GlobalRegistryTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_clearing(self): - """Validates that clearing the registry gives us a new registry.""" - _clear_global_registry() - self.assertNotEqual(self.registry, sweet_pickle.get_global_registry()) - - def test_registry_starts_empty(self): - """Validates that the registry is starting empty for each test.""" - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - - def test_returns_singleton(self): - """Validates that the getter returns the same global registry""" - # Just try it a few times. - self.assertEqual(self.registry, sweet_pickle.get_global_registry()) - self.assertEqual(self.registry, sweet_pickle.get_global_registry()) - self.assertEqual(self.registry, sweet_pickle.get_global_registry()) From 28f236eedbc1791a6f59e7ea1075ced002e944fc Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Mon, 23 Nov 2020 14:35:56 -0700 Subject: [PATCH 07/14] flake8 --- apptools/persistence/tests/test_two_stage_unpickler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apptools/persistence/tests/test_two_stage_unpickler.py b/apptools/persistence/tests/test_two_stage_unpickler.py index 0231dac85..2b7d202d3 100644 --- a/apptools/persistence/tests/test_two_stage_unpickler.py +++ b/apptools/persistence/tests/test_two_stage_unpickler.py @@ -14,6 +14,7 @@ import io import random +import re import pickle import unittest @@ -21,6 +22,7 @@ ######################################## + # Usecase1: generic case class A(object): def __init__(self, b=None): @@ -93,7 +95,6 @@ def test_generic(self): ######################################## # Usecase2: Toy Application -import re class StringFinder(object): From ba44f1c3ff7d5f149d69f5ef04d80fca6ecc562d Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 24 Nov 2020 11:16:21 -0700 Subject: [PATCH 08/14] convert test_class_mapping to use persistence --- .../persistence/tests/test_class_mapping.py | 79 +++++++++++++++ .../sweet_pickle/tests/test_class_mapping.py | 99 ------------------- 2 files changed, 79 insertions(+), 99 deletions(-) create mode 100644 apptools/persistence/tests/test_class_mapping.py delete mode 100644 apptools/sweet_pickle/tests/test_class_mapping.py diff --git a/apptools/persistence/tests/test_class_mapping.py b/apptools/persistence/tests/test_class_mapping.py new file mode 100644 index 000000000..2e63e78c2 --- /dev/null +++ b/apptools/persistence/tests/test_class_mapping.py @@ -0,0 +1,79 @@ +# ----------------------------------------------------------------------------- +# +# Copyright (c) 2006 by Enthought, Inc. +# All rights reserved. +# +# Author: Dave Peterson +# +# ----------------------------------------------------------------------------- + +""" Tests the class mapping functionality of the enthought.pickle + framework. +""" + +# Standard library imports. +import io +import pickle +import unittest + +# Enthought library imports +from apptools.persistence.versioned_unpickler import VersionedUnpickler +from apptools.persistence.updater import Updater + + +############################################################################## +# Classes to use within the tests +############################################################################## + + +class Foo: + pass + + +class Bar: + pass + + +class Baz: + pass + + +############################################################################## +# class 'ClassMappingTestCase' +############################################################################## + + +class ClassMappingTestCase(unittest.TestCase): + """Tests the class mapping functionality of the apptools.sweet_pickle + framework. + """ + + ########################################################################## + # 'TestCase' interface + ########################################################################## + + ### public interface ##################################################### + + def test_unpickled_class_mapping(self): + + class TestUpdater(Updater): + def __init__(self): + self.refactorings = { + (Foo.__module__, Foo.__name__): (Bar.__module__, Bar.__name__), + (Bar.__module__, Bar.__name__): (Baz.__module__, Baz.__name__), + } + self.setstates = {} + + # Validate that unpickling the first class gives us an instance of + # the second class. + start = Foo() + test_file = io.BytesIO(pickle.dumps(start, 2)) + end = VersionedUnpickler(test_file, updater=TestUpdater()).load() + self.assertIsInstance(end, Bar) + + # Validate that unpickling the second class gives us an instance of + # the third class. + start = Bar() + test_file = io.BytesIO(pickle.dumps(start, 2)) + end = VersionedUnpickler(test_file, updater=TestUpdater()).load() + self.assertIsInstance(end, Baz) diff --git a/apptools/sweet_pickle/tests/test_class_mapping.py b/apptools/sweet_pickle/tests/test_class_mapping.py deleted file mode 100644 index e9170d002..000000000 --- a/apptools/sweet_pickle/tests/test_class_mapping.py +++ /dev/null @@ -1,99 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -# ----------------------------------------------------------------------------- - -""" Tests the class mapping functionality of the enthought.pickle - framework. -""" - -# Standard library imports. -import unittest - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry - - -############################################################################## -# Classes to use within the tests -############################################################################## - -# Need complete package name so that mapping matches correctly. -# The problem here is the Python loader that will load the same module with -# multiple names in sys.modules due to relative naming. Nice. -from apptools.sweet_pickle.tests.class_mapping_classes import Foo, Bar, Baz - -############################################################################## -# class 'ClassMappingTestCase' -############################################################################## - - -class ClassMappingTestCase(unittest.TestCase): - """Tests the class mapping functionality of the apptools.sweet_pickle - framework. - """ - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - ########################################################################## - # 'ClassMappingTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_infinite_loop_detection(self): - """Validates that the class mapping framework detects infinite - loops of class mappings. - """ - # Add mappings to the registry - self.registry.add_mapping_to_class(Foo.__module__, Foo.__name__, Bar) - self.registry.add_mapping_to_class(Bar.__module__, Bar.__name__, Baz) - self.registry.add_mapping_to_class(Baz.__module__, Baz.__name__, Foo) - - # Validate that an exception is raised when trying to unpickle an - # instance anywhere within the circular definition. - def fn(o): - sweet_pickle.loads(sweet_pickle.dumps(o)) - - self.assertRaises(sweet_pickle.UnpicklingError, fn, Foo()) - self.assertRaises(sweet_pickle.UnpicklingError, fn, Bar()) - self.assertRaises(sweet_pickle.UnpicklingError, fn, Baz()) - - def test_unpickled_class_mapping(self): - - # Add the mappings to the registry - self.registry.add_mapping_to_class(Foo.__module__, Foo.__name__, Bar) - self.registry.add_mapping_to_class(Bar.__module__, Bar.__name__, Baz) - - # Validate that unpickling the first class gives us an instance of - # the third class. - start = Foo() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - - # Validate that unpickling the second class gives us an instance of - # the third class. - start = Bar() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) From 6b057bb7ba392b98bcbd6454cf427bbf5a897815 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 24 Nov 2020 11:51:13 -0700 Subject: [PATCH 09/14] flake8 --- apptools/persistence/tests/test_class_mapping.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apptools/persistence/tests/test_class_mapping.py b/apptools/persistence/tests/test_class_mapping.py index 2e63e78c2..762992bbc 100644 --- a/apptools/persistence/tests/test_class_mapping.py +++ b/apptools/persistence/tests/test_class_mapping.py @@ -59,8 +59,10 @@ def test_unpickled_class_mapping(self): class TestUpdater(Updater): def __init__(self): self.refactorings = { - (Foo.__module__, Foo.__name__): (Bar.__module__, Bar.__name__), - (Bar.__module__, Bar.__name__): (Baz.__module__, Baz.__name__), + (Foo.__module__, Foo.__name__): + (Bar.__module__, Bar.__name__), + (Bar.__module__, Bar.__name__): + (Baz.__module__, Baz.__name__), } self.setstates = {} From ef2cd50c82bc1075529f009f148a2b33ec0867a2 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 24 Nov 2020 12:45:55 -0700 Subject: [PATCH 10/14] update test_state_function to use persistence --- .../tests/test_state_function.py | 69 ++++++++----------- 1 file changed, 28 insertions(+), 41 deletions(-) rename apptools/{sweet_pickle => persistence}/tests/test_state_function.py (69%) diff --git a/apptools/sweet_pickle/tests/test_state_function.py b/apptools/persistence/tests/test_state_function.py similarity index 69% rename from apptools/sweet_pickle/tests/test_state_function.py rename to apptools/persistence/tests/test_state_function.py index ab0f2eacb..5b73bfd36 100644 --- a/apptools/sweet_pickle/tests/test_state_function.py +++ b/apptools/persistence/tests/test_state_function.py @@ -12,12 +12,15 @@ """ # Standard library imports. +import io +import pickle import unittest import logging # Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry +from apptools.persistence.versioned_unpickler import VersionedUnpickler +from apptools.persistence.updater import Updater + from traits.api import Bool, Float, HasTraits, Int, Str @@ -38,7 +41,7 @@ ############################################################################## -def bar_state_function(state): +def bar_state_function(self, state): for old, new in [("b1", "b2"), ("f1", "f2"), ("i1", "i2"), ("s1", "s2")]: state[new] = state[old] del state[old] @@ -46,6 +49,18 @@ def bar_state_function(state): return state +class TestUpdater(Updater): + def __init__(self): + self.refactorings = { + (Foo.__module__, Foo.__name__): + (Bar.__module__, Bar.__name__), + (Bar.__module__, Bar.__name__): + (Baz.__module__, Baz.__name__), + } + self.setstates = {} + + + ############################################################################## # class 'StateFunctionTestCase' ############################################################################## @@ -68,15 +83,9 @@ def setUp(self): Overridden here to ensure each test starts with an empty global registry. """ - # Clear the global registry - _clear_global_registry() - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() + self.updater = TestUpdater() - # Add the class mappings to the registry - self.registry.add_mapping_to_class(Foo.__module__, Foo.__name__, Bar) - self.registry.add_mapping_to_class(Bar.__module__, Bar.__name__, Baz) ########################################################################## # 'StateFunctionTestCase' interface @@ -89,47 +98,25 @@ def test_normal_setstate(self): there are no registered state functions in the class chain. """ # Validate that unpickling the first class gives us an instance of - # the third class with the appropriate attribute values. It will have + # the second class with the appropriate attribute values. It will have # the default Foo values (because there is no state function to move - # them) and also the default Baz values (since they inherit the + # them) and also the default Bar values (since they inherit the # trait defaults because nothing overwrote the values.) start = Foo() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) + test_file = io.BytesIO(pickle.dumps(start, 2)) + end = VersionedUnpickler(test_file, updater=TestUpdater()).load() + self.assertIsInstance(end, Bar) self._assertAttributes(end, 1, (False, 1, 1, "foo")) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (False, 3, 3, "baz")) + self._assertAttributes(end, 2, (True, 2, 2, "bar")) + self._assertAttributes(end, 3, None) # Validate that unpickling the second class gives us an instance of # the third class with the appropriate attribute values. It will have # only the Baz attributes with the Bar values (since the __setstate__ # on Baz converted the Bar attributes to Baz attributes.) start = Bar() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (True, 2, 2, "bar")) - - def test_unpickled_chain_functionality(self): - """Validates that the registered state functions are used when - unpickling. - """ - # Add the state function to the registry - self.registry.add_state_function_for_class(Bar, 2, bar_state_function) - - # Validate that unpickling the first class gives us an instance of - # the third class with the appropriate attribute values. - start = Foo() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) - self.assertIsInstance(end, Baz) - self._assertAttributes(end, 1, None) - self._assertAttributes(end, 2, None) - self._assertAttributes(end, 3, (False, 1, 1, "foo")) - - # Validate that unpickling the second class gives us an instance of - # the third class. - start = Bar() - end = sweet_pickle.loads(sweet_pickle.dumps(start)) + test_file = io.BytesIO(pickle.dumps(start, 2)) + end = VersionedUnpickler(test_file, updater=TestUpdater()).load() self.assertIsInstance(end, Baz) self._assertAttributes(end, 2, None) self._assertAttributes(end, 3, (True, 2, 2, "bar")) From 595a7ef6d4e3117713e7e0aaff7f6d77c66db2a9 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 24 Nov 2020 12:51:34 -0700 Subject: [PATCH 11/14] delete apptools.sweet_pickle --- .../tests/state_function_classes.py | 2 - .../persistence/tests/test_state_function.py | 7 +- apptools/sweet_pickle/__init__.py | 176 ------ apptools/sweet_pickle/global_registry.py | 102 ---- apptools/sweet_pickle/placeholder.py | 26 - .../tests/class_mapping_classes.py | 24 - apptools/sweet_pickle/tests/test_updater.py | 342 ----------- apptools/sweet_pickle/updater.py | 348 ----------- apptools/sweet_pickle/versioned_unpickler.py | 562 ------------------ setup.cfg | 1 - 10 files changed, 4 insertions(+), 1586 deletions(-) rename apptools/{sweet_pickle => persistence}/tests/state_function_classes.py (92%) delete mode 100644 apptools/sweet_pickle/__init__.py delete mode 100644 apptools/sweet_pickle/global_registry.py delete mode 100644 apptools/sweet_pickle/placeholder.py delete mode 100644 apptools/sweet_pickle/tests/class_mapping_classes.py delete mode 100644 apptools/sweet_pickle/tests/test_updater.py delete mode 100644 apptools/sweet_pickle/updater.py delete mode 100644 apptools/sweet_pickle/versioned_unpickler.py diff --git a/apptools/sweet_pickle/tests/state_function_classes.py b/apptools/persistence/tests/state_function_classes.py similarity index 92% rename from apptools/sweet_pickle/tests/state_function_classes.py rename to apptools/persistence/tests/state_function_classes.py index bc5bab039..fc882ce69 100644 --- a/apptools/sweet_pickle/tests/state_function_classes.py +++ b/apptools/persistence/tests/state_function_classes.py @@ -11,8 +11,6 @@ import logging # Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry from traits.api import Bool, Float, HasTraits, Int, Str diff --git a/apptools/persistence/tests/test_state_function.py b/apptools/persistence/tests/test_state_function.py index 5b73bfd36..dee6a38bc 100644 --- a/apptools/persistence/tests/test_state_function.py +++ b/apptools/persistence/tests/test_state_function.py @@ -7,8 +7,9 @@ # # ----------------------------------------------------------------------------- -""" Tests the state function functionality of the apptools.sweet_pickle - framework. +""" These tests were originally for the the state function functionality of the +now deleted apptools.sweet_pickle framework. They have been modified here to +use apptools.persistence instead. """ # Standard library imports. @@ -34,7 +35,7 @@ # Need complete package name so that mapping matches correctly. # The problem here is the Python loader that will load the same module with # multiple names in sys.modules due to relative naming. Nice. -from apptools.sweet_pickle.tests.state_function_classes import Foo, Bar, Baz +from apptools.persistence.tests.state_function_classes import Foo, Bar, Baz ############################################################################## # State functions to use within the tests diff --git a/apptools/sweet_pickle/__init__.py b/apptools/sweet_pickle/__init__.py deleted file mode 100644 index 87573c6b9..000000000 --- a/apptools/sweet_pickle/__init__.py +++ /dev/null @@ -1,176 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -# ----------------------------------------------------------------------------- - -""" A pickle framework that supports unpickling of refactored versions of - old classes. Part of the AppTools project of the Enthought Tool Suite. - - Beyond refactoring support, there are additional useful capabilities of - this package: - - - HasTraits objects are fully pickled when sweet_pickle.dump() or - sweet_pickle.dumps() are called. This means every trait's value - is pickled explicitly. This allows improved self-consistency as - values within unpickled objects correspond to the default trait - values of the instance that was pickled rather than the default - values of the current class definition. - - Multiple updaters can be registered to update state for any version - of a class. This is extremely useful when an instance being - pickled and unpickled is the result of an addition of one or more - Trait Categories to a class. In that case, the Category contributor - can also register a state function to update the Category's traits - through versioning. - - - We have duplicated the API of the Python 'pickle' module within this - package. Thus, users can simply import sweet_pickle in places where - they previously used pickle or cPickle. For example:: - - import cPickle ---> import apptools.sweet_pickle as pickle - s = pickle.dumps(obj) s = pickle.dumps(obj) - pickle.loads(s) pickle.loads(s) - - Pickles generated by this package *can be* unpickled by standard pickle - or cPickle, though you lose the benefit of the refactoring support during - unpickling. - - As a result of the above, this package is a drop-in replacement for the - Python 'pickle' module or 'cPickle' module and should be safe to use even - if you never version or refactor your own classes. In fact, we STRONGLY - RECOMMEND that you use this framework for all of your pickle needs because - you never know when one of the classes you encounter during unpickling has - been versioned or otherwise refactored by someone else. - - See module 'pickle' for more basic information about pickling. - - - The most common way to benefit from the versioning capabilities of this - framework is to register class mappings and state modification functions - with the global updater registry (more detail is below.) However, you may - also choose to explicitly instantiate an Unpickler and provide it with your - own explicit definition of class mappings and state modification functions. - We do not provide any help on the latter at this time. - - You can register class mappings and state modification functions with the - global updater registry using the methods provided on the Updater instance - that is the global registry. You can get this instance by calling the - 'get_global_registry' function exposed through this package's namespace. - - This framework has been designed so that you can register your class - mappings and state modification functions during the import of the module - or package that contained the original class. However, you may also - register your mappings and functions at any point such that the they are - known to the framework prior to, or become known during the attempt to, - unpickle the class they modify. - - The framework will call a __setstate__ method on the final target class - of any unpickled instance. It will not call __setstate__ methods on any - beginning or intermediate classes within a chain of class mappings. - - - A class mapping is used to redirect the unpickling of one class to return - an instantiation of another class. The classes can be in different - modules, and the modules in different packages. Mappings can be chained. - For example, given the mappings:: - - foo.bar.Bar --> foo.baz.Baz - foo.baz.Baz --> foo.Foo - - An attempt to unpickle a foo.bar.Bar would actually generate a foo.Foo - instance. - - A state modification function is called during the execution of the - __setstate__ method during unpickling of an object of the type and version - for which it was registered for. The function must accept a single - argument which is a state dictionary and must then return the modified - state dictionary. Additionally, the function should change the version - variable within the state to represent the version the new state - represents. (The framework will assume an increment of one if this is - not done.) The framework ensures that state modification functions - are chained appropriately to convert through multiple versions and/or class - mappings. - - Note that refactorings that cause classes to be completed removed from - the source code can be supported, without breaking unpickling of object - hierarchies that include an instace of that class, by adding a mapping - to the Placeholder class in the placeholder module. -""" - -############################################################################## -# Implement the Python pickle package API -############################################################################## - -# Expose our custom pickler as the standard Unpickler -from .versioned_unpickler import VersionedUnpickler as Unpickler - -# Use our custom unpickler to load from files -def load(file, max_pass=-1): - return Unpickler(file).load(max_pass) - - -# Use our custom unpickler to load from strings -def loads(str, max_pass=-1): - from io import BytesIO - - file = BytesIO(str) - - return Unpickler(file).load(max_pass) - - -# We don't customize the Python pickler, though we do use the cPickle module -# for improved performance. -from pickle import Pickler - -# Implement the dump and dumps methods so that all traits in a HasTraits object -# get included in the pickle. -def dump(obj, file, protocol=2): - _flush_traits(obj) - from pickle import dump as d - - return d(obj, file, protocol) - - -def dumps(obj, protocol=2): - _flush_traits(obj) - from pickle import dumps as ds - - return ds(obj, protocol) - - -# We don't customize exceptions so just map to the Python pickle package -from pickle import PickleError, PicklingError, UnpicklingError - - -############################################################################## -# Allow retrieval of the global registry -############################################################################## - -from .global_registry import get_global_registry - - -############################################################################## -# Expose our Updater class so users can explicitly create their own. -############################################################################## - -from .updater import Updater - - -############################################################################## -# Method to ensure that all traits on a has traits object are included in a -# pickle. -############################################################################## - - -def _flush_traits(obj): - if hasattr(obj, "trait_names"): - for name, value in obj.traits().items(): - if value.type == "trait": - try: - getattr(obj, name) - except AttributeError: - pass diff --git a/apptools/sweet_pickle/global_registry.py b/apptools/sweet_pickle/global_registry.py deleted file mode 100644 index de52eeda4..000000000 --- a/apptools/sweet_pickle/global_registry.py +++ /dev/null @@ -1,102 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -# ----------------------------------------------------------------------------- - -""" Manages a singleton updater that acts as a global registry. - - Our goal is to enable the apptools.sweet_pickle framework to - understand how pickled data should be treated during unpickling so - that the resulting object hierarchy reflects the current versions - of the object's classes -AND- that this should work no matter who - is doing the unpickling. This requires a global registry since - there is no way to know in advance what objects are contained within - a given pickle file, and thus no way to gather the required - information to know how to treat the pickle data during unpickling. - - For example, a pickle of an Envisage project may contain many - custom classes, instantiated at the behest of various plugins, - which have gone through various versionings and refactorings. But - it is the project plugin that needs to unpickle these objects to - 'load' a project, not the other plugins that added those custom - class instances into the project. - - - This registry is used by the apptools.sweet_pickle framework's - unpickler only by default. That is, only if no updater was - explicitly provided. - - - It is important that users interact with the registry through the - provided methods. If they do not, then the reference they receive - will be the one that was in place at the time of the import which - may or MAY NOT be the current repository due to the way this - framework manages the repository. -""" - -try: - import _thread -except ImportError: - import _dummy_thread - - -############################################################################## -# function 'get_global_registry' -############################################################################## - - -def get_global_registry(): - """Returns the global registry in a manner that allows for lazy - instantiation. - """ - global _global_registry, _global_registry_lock - - # Do we need to create the registry? - if _global_registry is None: - - # We can only do this safely in a threaded situation by using a lock. - # Note that the previous check for None doesn't guarantee we are - # the only one trying to create an instance, so, we'll check for none - # again once we acquire the lock and then only create the singleton - # if there still isn't an instance. - _global_registry_lock.acquire() - try: - if _global_registry is None: - from .updater import Updater - - _global_registry = Updater() - finally: - _global_registry_lock.release() - - return _global_registry - - -############################################################################## -# private function '_clear_global_registry' -############################################################################## - - -def _clear_global_registry(): - """Clears out the current global registry. - - This exists purely to allow testing of the global registry and the - apptools.sweet_pickle framework. THIS METHOD SHOULD NEVER BE - CALLED DURING NORMAL OPERATIONS! - """ - global _global_registry - _global_registry = None - - -############################################################################## -# private, but global, variables -############################################################################## - -# The global singleton updater -_global_registry = None - -# The lock used to make access to the global singleton thread safe -_global_registry_lock = _thread.allocate_lock() diff --git a/apptools/sweet_pickle/placeholder.py b/apptools/sweet_pickle/placeholder.py deleted file mode 100644 index 5e8c2b9a9..000000000 --- a/apptools/sweet_pickle/placeholder.py +++ /dev/null @@ -1,26 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Vibha Srinivasan -# -# ----------------------------------------------------------------------------- - -""" An empty class that serves as a placeholder to map a class to when - that class has been deleted as a result of a refactoring. -""" - -# Enthought library imports -from traits.api import HasTraits - - -############################################################################## -# class 'PlaceHolder' -############################################################################## - - -class PlaceHolder(HasTraits): - """An empty class that serves as a placeholder to map a class to when - that class has been deleted as a result of a refactoring. - """ diff --git a/apptools/sweet_pickle/tests/class_mapping_classes.py b/apptools/sweet_pickle/tests/class_mapping_classes.py deleted file mode 100644 index d91b5a0c4..000000000 --- a/apptools/sweet_pickle/tests/class_mapping_classes.py +++ /dev/null @@ -1,24 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -# ----------------------------------------------------------------------------- - -############################################################################## -# Classes to use within the tests -############################################################################## - - -class Foo: - pass - - -class Bar: - pass - - -class Baz: - pass diff --git a/apptools/sweet_pickle/tests/test_updater.py b/apptools/sweet_pickle/tests/test_updater.py deleted file mode 100644 index 52a02d73b..000000000 --- a/apptools/sweet_pickle/tests/test_updater.py +++ /dev/null @@ -1,342 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# -# ----------------------------------------------------------------------------- - -""" Tests the updater functionality of the sweet_pickle framework. -""" - -# Standard library imports. -import unittest -import logging - -# Enthought library imports -import apptools.sweet_pickle as sweet_pickle -from apptools.sweet_pickle.global_registry import _clear_global_registry - - -logger = logging.getLogger(__name__) - - -############################################################################## -# class 'UpdaterTestCase' -############################################################################## - - -class UpdaterTestCase(unittest.TestCase): - """Tests the updater functionality of the sweet_pickle framework.""" - - ########################################################################## - # 'TestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def setUp(self): - """Creates the test fixture. - - Overridden here to ensure each test starts with an empty global - registry. - """ - # Clear the global registry - _clear_global_registry() - - # Cache a reference to the new global registry - self.registry = sweet_pickle.get_global_registry() - - ########################################################################## - # 'UpdaterTestCase' interface - ########################################################################## - - ### public interface ##################################################### - - def test_add_mapping(self): - """Validates the behavior of the add_mapping function.""" - # Add a single mapping and validate that all it did was add a class - # mapping and that the mapping has the expected values. - key = ("foo", "Foo") - value = ("bar", "Bar") - self.registry.add_mapping(key[0], key[1], value[0], value[1]) - self._validate_exactly_one_class_map(key, value) - - # Overwrite with a new mapping and validate the state is what we - # expect. - value = ("baz", "Baz") - self.registry.add_mapping(key[0], key[1], value[0], value[1]) - self._validate_exactly_one_class_map(key, value) - - def test_add_mapping_to_class(self): - """Validates the behavior of the add_mapping_to_class function.""" - # Add a single mapping and validate that all it did was add a class - # mapping and that the mapping has the expected values. - key = ("foo", "Foo") - - class Bar: - pass - - value = (Bar.__module__, "Bar") - self.registry.add_mapping_to_class(key[0], key[1], Bar) - self._validate_exactly_one_class_map(key, value) - - # Overwrite with a new mapping and validate the state is what we - # expect. - class Baz: - pass - - value = (Baz.__module__, "Baz") - self.registry.add_mapping_to_class(key[0], key[1], Baz) - self._validate_exactly_one_class_map(key, value) - - def test_add_mappings(self): - """Validates the behavior of the add_mappings function.""" - # Add a single mapping and validate that all it did was add a class - # mapping and that the mapping has the expected values - key = ("foo", "Foo") - value = ("bar", key[1]) - names = [key[1]] - self.registry.add_mappings(key[0], value[0], names) - self._validate_exactly_one_class_map(key, value) - - # Add multiple mappings and validate that the registry has the expected - # values. - key = ("foo", "Foo") - value = ("bar", "Bar") - names = ["Foo", "Bar", "Baz", "Enthought"] - self.registry.add_mappings(key[0], value[0], names) - self.assertEqual(len(names), len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - items = [] - for n in names: - items.append(((key[0], n), (value[0], n))) - self._validate_class_map_contents(items) - - def test_add_state_function(self): - """Validates the behavior of the add_state_function function.""" - # Add a single function and validate that all it did was add a state - # mapping and that the mapping has the expected values. - key = ("foo", "Foo", 1) - - def fn(): - pass - - self.registry.add_state_function(key[0], key[1], key[2], fn) - self._validate_exactly_one_state_function(key, [fn]) - - # Add an additional function for the same state and validate the state - # is what we expect. - def fn2(): - pass - - self.registry.add_state_function(key[0], key[1], key[2], fn2) - self._validate_exactly_one_state_function(key, [fn, fn2]) - - # Add a state function for another version of the same class and - # validate that all the values are as expected. - key2 = ("foo", "Foo", 2) - self.registry.add_state_function(key2[0], key2[1], key2[2], fn2) - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(2, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self._validate_state_function_contents( - [(key, [fn, fn2]), (key2, [fn2])], {(key[0], key[1]): 3} - ) - - def test_add_state_function_for_class(self): - """Validates the behavior of the add_state_function_for_class function.""" - # Add a single function and validate that all it did was add a state - # mapping and that the mapping has the expected values. - class Bar: - pass - - key = (Bar.__module__, "Bar", 1) - - def fn(): - pass - - self.registry.add_state_function_for_class(Bar, key[2], fn) - self._validate_exactly_one_state_function(key, [fn]) - - # Add an additional function for the same state and validate the state - # is what we expect. - def fn2(): - pass - - self.registry.add_state_function_for_class(Bar, key[2], fn2) - self._validate_exactly_one_state_function(key, [fn, fn2]) - - # Add a state function for another class and validate that all the - # values are as expected. - class Baz: - pass - - key2 = (Baz.__module__, "Baz", 2) - self.registry.add_state_function_for_class(Baz, key2[2], fn2) - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(2, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(2, len(self.registry._state_function_classes)) - self._validate_state_function_contents( - [(key, [fn, fn2]), (key2, [fn2])], - {(key[0], key[1]): 2, (key2[0], key2[1]): 1}, - ) - - def test_merge_updater(self): - """Validates the behavior of the merge_updater function.""" - # Merge in one update and validate the state of the registry is as - # expected. - def fn1(): - pass - - updater = sweet_pickle.Updater( - class_map={ - ("foo", "Foo"): ("foo.bar", "Foo"), - }, - state_functions={ - ("foo", "Foo", 1): [fn1], - }, - version_attribute_map={ - ("foo", "Foo"): "version", - }, - ) - self.registry.merge_updater(updater) - self.assertEqual(1, len(self.registry.class_map)) - self.assertEqual(1, len(self.registry.state_functions)) - self.assertEqual(1, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self._validate_class_map_contents(list(updater.class_map.items())) - counts = {("foo", "Foo"): 1} - self._validate_state_function_contents( - list(updater.state_functions.items()), counts - ) - - # Merge in a second updater and validate the state of the registry is - # as expected. - def fn2(): - pass - - updater2 = sweet_pickle.Updater( - class_map={ - ("foo.bar", "Foo"): ("bar", "Bar"), - ("bar", "Bar"): ("foo.bar.baz", "Baz"), - }, - state_functions={ - ("foo", "Foo", 1): [fn2], - ("foo", "Foo", 2): [fn2], - }, - version_attribute_map={ - ("foo.bar", "Foo"): "_version", - }, - ) - self.registry.merge_updater(updater2) - self.assertEqual(3, len(self.registry.class_map)) - self.assertEqual(2, len(self.registry.state_functions)) - self.assertEqual(2, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self._validate_class_map_contents( - list(updater.class_map.items()) + list(updater2.class_map.items()) - ) - counts = {("foo", "Foo"): 3} - self._validate_state_function_contents( - [(("foo", "Foo", 1), [fn1, fn2]), (("foo", "Foo", 2), [fn2])], - counts, - ) - - def test_registry_starts_empty(self): - """Validates that the registry is starting empty for each test.""" - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - - ### protected interface ################################################## - - def _validate_class_map_contents(self, items): - """Validates that the registry's class_map contains the specified - items. - """ - for key, value in items: - self.assertEqual( - True, - key in self.registry.class_map, - "Key " + str(key) + " not in class_map", - ) - self.assertEqual( - value, - self.registry.class_map[key], - str(value) - + " != " - + str(self.registry.class_map[key]) - + " for key " - + str(key), - ) - self.assertEqual( - True, - self.registry.has_class_mapping(key[0], key[1]), - "Registry reports no class mapping for key " + str(key), - ) - - def _validate_exactly_one_class_map(self, key, value): - """Validates that the registry has exactly one class_map entry - with the specified key and value. - """ - self.assertEqual(1, len(self.registry.class_map)) - self.assertEqual(0, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(0, len(self.registry._state_function_classes)) - self._validate_class_map_contents([(key, value)]) - - def _validate_exactly_one_state_function(self, key, value): - """Validates that the registry has exactly one state_function entry - with the specified key and value. - """ - self.assertEqual(0, len(self.registry.class_map)) - self.assertEqual(1, len(self.registry.state_functions)) - self.assertEqual(0, len(self.registry.version_attribute_map)) - self.assertEqual(1, len(self.registry._state_function_classes)) - self.assertEqual(key, list(self.registry.state_functions.keys())[0]) - self.assertEqual(value, self.registry.state_functions[key]) - classes_key = (key[0], key[1]) - self.assertEqual( - classes_key, list(self.registry._state_function_classes.keys())[0] - ) - self.assertEqual( - len(value), self.registry._state_function_classes[classes_key] - ) - - def _validate_state_function_contents(self, items, counts): - """Validates that the registry's state functions contains the - specified items and the class count matches the specified count. - """ - for key, value in items: - self.assertEqual( - True, - key in self.registry.state_functions, - "Key " + str(key) + " not in state functions", - ) - self.assertEqual( - value, - self.registry.state_functions[key], - str(value) - + " != " - + str(self.registry.state_functions[key]) - + " for key " - + str(key), - ) - self.assertEqual( - True, - self.registry.has_state_function(key[0], key[1]), - "Registry reports no state function for key " + str(key), - ) - - classes_key = (key[0], key[1]) - count = counts[classes_key] - self.assertEqual( - count, self.registry._state_function_classes[classes_key] - ) diff --git a/apptools/sweet_pickle/updater.py b/apptools/sweet_pickle/updater.py deleted file mode 100644 index b8759ddae..000000000 --- a/apptools/sweet_pickle/updater.py +++ /dev/null @@ -1,348 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2005, 2006 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# Author: Duncan Child -# -# ----------------------------------------------------------------------------- - -""" A record of refactorings to be performed during unpickling of objects. -""" - -# Standard library imports -import logging - -# Enthought library imports -from traits.api import Dict, HasPrivateTraits, Int, List, Tuple, Str - - -logger = logging.getLogger(__name__) - - -############################################################################## -# class 'Updater' -############################################################################## - - -class Updater(HasPrivateTraits): - """A record of refactorings to be performed during unpickling of objects.""" - - ########################################################################## - # Traits - ########################################################################## - - ### public 'Updater' interface ########################################### - - # Mappings from a pickled class to a class it should be unpickled as. - # - # The keys are a tuple of the source class's module and class names in - # that order. The values are the target class's module and class names - # in that order. - class_map = Dict(Tuple(Str, Str), Tuple(Str, Str)) - - # State functions that should be called to convert state from one version - # of a class to another. - # - # The keys are a tuple of the class's module name, class name, and version - # in that order. The values are a list of functions to be called during - # unpickling to do the state conversion. Note that the version in the - # key represents the version that the function converts *TO*. - state_functions = Dict(Tuple(Str, Str, Int), List) - - # Our record of the attribute that records the version number for a - # specific class. If no record is found for a given class, then the - # default value is used instead -- see '_default_version_attribute'. - # - # The key is a tuple of the class's module name and class name in that - # order. The value is the name of the version attribute. - version_attribute_map = Dict(Tuple(Str, Str), Str) - - ### protected 'Updater' interface ######################################## - - # The default name of the attribute that declares the version of a class - # or class instance. - _default_version_attribute = "_enthought_pickle_version" - - # A record of which classes we have state functions for. - # - # The keys are a tuple of the class's module name and class name in that - # order. The values are reference counts. - _state_function_classes = Dict(Tuple(Str, Str), Int) - - ########################################################################## - # 'Updater' interface - ########################################################################## - - ### public interface ##################################################### - - def add_mapping( - self, source_module, source_name, target_module, target_name - ): - """Adds a mapping from the class with the source name in the source - module to the class with the target name in the target module. - """ - self.class_map[(source_module, source_name)] = ( - target_module, - target_name, - ) - - def add_mapping_to_class(self, source_module, source_name, target_class): - """Convenience method to add a mapping, from the class with the - source name in the source module to the target class. - """ - self.add_mapping( - source_module, - source_name, - target_class.__module__, - target_class.__name__, - ) - - def add_mappings(self, source_module, target_module, class_names): - """Adds mappings, from the specified source module to the specified - target module, for each of the class names in the specified - list. - """ - for name in class_names: - self.add_mapping(source_module, name, target_module, name) - - def add_state_function(self, module, name, target_version, function): - """Adds the specified function as a state function to be called to - convert an instance of the class with the specified name within - the specified module *TO* the specified version. - - Note that the framework handles calling of state functions to make - the smallest version jumps possible. - """ - key = (module, name, target_version) - list = self.state_functions.setdefault(key, []) - list = list[:] # Copy necessary because traits only recognizes list - # changes by list instance - not its contents. - list.append(function) - self.state_functions[key] = list - - def add_state_function_for_class(self, klass, target_version, function): - """Convenience method to add the specified function as a state - function to be called to convert an instance of the specified - class *TO* the specified version. - """ - self.add_state_function( - klass.__module__, klass.__name__, target_version, function - ) - - def declare_version_attribute(self, module, name, attribute_name): - """Adds the specified attribute name as the version attribute for the - class within the specified module with the specified name. - """ - self.version_attribute_map[(module, name)] = attribute_name - - def declare_version_attribute_for_class(self, klass, attribute_name): - """Covenience method to add the specified attribute name as the - version attribute for the specified class. - """ - self.declare_version_attribute( - klass.__module__, klass.__name__, attribute_name - ) - - def get_version_attribute(self, module, name): - """Returns the name of the version attribute for the class of the - specified name within the specified module. - """ - return self.version_attribute_map.get( - (module, name), self._default_version_attribute - ) - - def has_class_mapping(self, module, name): - """Returns True if this updater contains a class mapping for - the class identified by the specified module and class name. - """ - return (module, name) in self.class_map - - def has_state_function(self, module, name): - """Returns True if this updater contains any state functions for - the class identified by the specified module and class name. - """ - return (module, name) in self._state_function_classes - - def merge_updater(self, updater): - """Merges the mappings and state functions from the specified updater - into this updater. - """ - self.class_map.update(updater.class_map) - self.version_attribute_map.update(updater.version_attribute_map) - - # The state functions dictionary requires special processing because - # each value is a list and we don't just want to replace the existing - # list with only the new content. - for key, value in updater.state_functions.items(): - if isinstance(value, list) and len(value) > 0: - funcs = self.state_functions.setdefault(key, []) - funcs = funcs[ - : - ] # Copy necessary because traits only recognizes - # funcs changes by funcs instance - not its - # contents. - funcs.extend(value) - self.state_functions[key] = funcs - - ### trait handlers ####################################################### - - def _class_map_changed(self, old, new): - logger.debug( - "Detected class_map change from [%s] to [%s] in [%s]", - old, - new, - self, - ) - - def _class_map_items_changed(self, event): - for o in event.removed: - logger.debug( - "Detected [%s] removed from class_map in [%s]", o, self - ) - for k, v in event.changed.items(): - logger.debug( - "Detected [%s] changed from [%s] to [%s] in " - + "class_map in [%s]", - k, - v, - self.class_map[k], - self, - ) - for k, v in event.added.items(): - logger.debug( - "Detected mapping from [%s] to [%s] added to " - + "class_map in [%s]", - k, - v, - self, - ) - - def _state_functions_changed(self, old, new): - logger.debug( - "Detected state_functions changed from [%s] to [%s] " + "in [%s]", - old, - new, - self, - ) - - # Update our record of which classes we have state functions for. - # All of our old state functions are gone so we simply need to rescan - # the new functions. - self._state_function_classes.clear() - for key, value in new.items(): - module, name, version = key - klass_key = (module, name) - count = self._state_function_classes.setdefault(klass_key, 0) - self._state_function_classes[klass_key] = count + len(value) - - def _state_functions_items_changed(self, event): - # Decrement our reference counts for the classes we no longer - # have state functions for. If the reference count reaches zero, - # remove the record completely. - for k, v in event.removed.items(): - logger.debug( - "Detected [%s] removed from state_functions in [%s]", k, self - ) - - # Determine the new reference count of state functions for the - # class who the removed item was for. - module, name, version = k - key = (module, name) - count = self._state_function_classes[key] - len(v) - - # Store the new reference count. Delete the entry if it is zero. - if count < 0: - logger.warn( - "Unexpectedly reached negative reference count " - + "value of [%s] for [%s]", - count, - key, - ) - del self._state_function_classes[key] - elif count == 0: - del self._state_function_classes[key] - else: - self._state_function_classes[key] = count - - # Update our reference counts for changes to the list of functions - # for a specific class and version. The 'changed' dictionary's values - # are the old values. - for k, v in event.changed.items(): - value = self.state_functions[k] - logger.debug( - "Detected [%s] changed in state_functions from " - + "[%s] to [%s] in [%s]", - k, - v, - value, - self, - ) - - # Determine the new reference count as a result of the change. - module, name, version = k - key = (module, name) - count = self._state_function_classes[key] - len(v) + len(value) - - # Store the new reference count. Delete the entry if it is zero. - if count < 0: - logger.warn( - "Unexpectedly reached negative reference count " - + "value of [%s] for [%s]", - count, - key, - ) - del self._state_function_classes[key] - elif count == 0: - del self._state_function_classes[key] - else: - self._state_function_classes[key] = count - - # Update our reference counts for newly registered state functions. - for k, v in event.added.items(): - logger.debug( - "Detected mapping of [%s] to [%s] added to " - + "state_functions in [%s]", - k, - v, - self, - ) - - # Determine the new reference count as a result of the change. - module, name, version = k - key = (module, name) - count = self._state_function_classes.setdefault(key, 0) + len(v) - - # Store the new reference count - self._state_function_classes[key] = count - - def _version_attribute_map_changed(self, old, new): - logger.debug( - "Detected version_attribute_map change from [%s] " - + "to [%s] in [%s]", - old, - new, - self, - ) - - def _version_attribute_map_items_changed(self, event): - for o in event.removed: - logger.debug( - "Detected [%s] removed from version_attribute_map " - + "in [%s]", - o, - self, - ) - for o in event.changed: - logger.debug( - "Detected [%s] changed in version_attribute_map " + "in [%s]", - o, - self, - ) - for o in event.added: - logger.debug( - "Detected [%s] added to version_attribute_map in " + "[%s]", - o, - self, - ) diff --git a/apptools/sweet_pickle/versioned_unpickler.py b/apptools/sweet_pickle/versioned_unpickler.py deleted file mode 100644 index 69a5b31ac..000000000 --- a/apptools/sweet_pickle/versioned_unpickler.py +++ /dev/null @@ -1,562 +0,0 @@ -# ----------------------------------------------------------------------------- -# -# Copyright (c) 2005-2008 by Enthought, Inc. -# All rights reserved. -# -# Author: Dave Peterson -# Author: Duncan Child -# -# ----------------------------------------------------------------------------- -# The code for two-stage unpickling support has been taken from a PEP draft -# prepared by Dave Peterson and Prabhu Ramachandran. - -""" An unpickler that is tolerant of class refactorings, and implements a -two-stage pickling process to make it possible to unpickle complicated Python -object hierarchies where the unserialized state of an object depends on the -state of other objects in the same pickle. -""" - -# Standard library imports. -import sys -import logging -from os import path -from types import GeneratorType - -if sys.version_info[0] >= 3: - from pickle import _Unpickler as Unpickler -else: - from pickle import Unpickler - -from pickle import UnpicklingError, BUILD - -# Enthought library imports -from traits.api import HasTraits, Instance - - -# Setup a logger for this module -logger = logging.getLogger(__name__) - - -############################################################################## -# constants -############################################################################## - -# The name we backup the original setstate method to. -_BACKUP_NAME = "__enthought_sweet_pickle_original_setstate__" - -# The name of the setstate method we hook -_SETSTATE_NAME = "__setstate__" - -# The name we store our unpickling data under. -_UNPICKLER_DATA = "__enthought_sweet_pickle_unpickler__" - - -############################################################################## -# function '__replacement_setstate__' -############################################################################## - - -def __replacement_setstate__(self, state): - """Called to enable an unpickler to modify the state of this instance.""" - # Retrieve the unpickling information and use it to let the unpickler - # modify our state. - unpickler, module, name = getattr(self, _UNPICKLER_DATA) - state = unpickler.modify_state(self, state, module, name) - - # If we were given a state, apply it to this instance now. - if state is not None: - - # Save our state - logger.debug("Final state: %s", state) - self.__dict__.update(state) - - -############################################################################## -# function 'load_build_with_meta_data' -############################################################################## - - -def load_build_with_meta_data(self): - """Called prior to the actual load_build() unpickling method which primes - the state dictionary with meta-data. - """ - - # Access the state object and check if it is a dictionary (state may also be - # a tuple, which is used for other unpickling build operations). Proceed to - # the standard load_build() if the state obj is not a dict. - state = self.stack[-1] - if type(state) == dict: - - # If a file object is used, reference the file name - if hasattr(self._file, "name"): - pickle_file_name = path.abspath(self._file.name) - else: - pickle_file_name = "" - - # Add any meta-data needed by __setstate__() methods here... - state["_pickle_file_name"] = pickle_file_name - - # Call the standard load_build() method - return self.load_build() - - -############################################################################## -# class 'NewUnpickler' -############################################################################## -class NewUnpickler(Unpickler): - """An unpickler that implements a two-stage pickling process to make it - possible to unpickle complicated Python object hierarchies where the - unserialized state of an object depends on the state of other objects in - the same pickle. - """ - - def load(self, max_pass=-1): - """Read a pickled object representation from the open file. - - Return the reconstituted object hierarchy specified in the file. - """ - # List of objects to be unpickled. - self.objects = [] - - # We overload the load_build method. - dispatch = self.dispatch - dispatch[BUILD[0]] = NewUnpickler.load_build - - # call the super class' method. - ret = Unpickler.load(self) - self.initialize(max_pass) - self.objects = [] - - # Reset the Unpickler's dispatch table. - dispatch[BUILD[0]] = Unpickler.load_build - return ret - - def initialize(self, max_pass): - # List of (object, generator) tuples that initialize objects. - generators = [] - - # Execute object's initialize to setup the generators. - for obj in self.objects: - if hasattr(obj, "__initialize__") and callable(obj.__initialize__): - ret = obj.__initialize__() - if isinstance(ret, GeneratorType): - generators.append((obj, ret)) - elif ret is not None: - raise UnpicklingError( - "Unexpected return value from " - "__initialize__. %s returned %s" % (obj, ret) - ) - - # Ensure a maximum number of passes - if max_pass < 0: - max_pass = len(generators) - - # Now run the generators. - count = 0 - while len(generators) > 0: - count += 1 - if count > max_pass: - not_done = [x[0] for x in generators] - msg = """Reached maximum pass count %s. You may have - a deadlock! The following objects are - uninitialized: %s""" % ( - max_pass, - not_done, - ) - raise UnpicklingError(msg) - for o, g in generators[:]: - try: - next(g) - except StopIteration: - generators.remove((o, g)) - - # Make this a class method since dispatch is a class variable. - # Otherwise, supposing the initial sweet_pickle.load call (which would - # have overloaded the load_build method) makes a pickle.load call at some - # point, we would have the dispatch still pointing to - # NewPickler.load_build whereas the object being passed in will be an - # Unpickler instance, causing a TypeError. - @classmethod - def load_build(cls, obj): - # Just save the instance in the list of objects. - if isinstance(obj, NewUnpickler): - obj.objects.append(obj.stack[-2]) - Unpickler.load_build(obj) - - -############################################################################## -# class 'VersionedUnpickler' -############################################################################## - - -class VersionedUnpickler(NewUnpickler, HasTraits): - """An unpickler that is tolerant of class refactorings. - - This class reads in a pickled file and applies the transforms - specified in its updater to generate a new hierarchy of objects - which are at the current version of the classes they are instances - of. - - Note that the creation of an updater is kept out of this class to - ensure that the class can be reused in different situations. - However, if no updater is provided during construction, then the - global registry updater will be used. - """ - - ########################################################################## - # Traits - ########################################################################## - - ### public 'VersionedUnpickler' interface ################################ - - # The updater used to modify the objects being unpickled. - updater = Instance("apptools.sweet_pickle.updater.Updater") - - ########################################################################## - # 'object' interface - ########################################################################## - - ### operator methods ##################################################### - - def __init__(self, file, **kws): - super(VersionedUnpickler, self).__init__(file) - - self._file = file - if self.updater is None: - from .global_registry import get_global_registry - - self.updater = get_global_registry() - logger.debug( - "VersionedUnpickler [%s] using Updater [%s]", self, self.updater - ) - - # Update the BUILD instruction to use an overridden load_build method - # NOTE: this is being disabled since, on some platforms, the object - # is replaced with a regular Unpickler instance, creating a traceback: - # AttributeError: Unpickler instance has no attribute '_file' - # ...not sure how this happens since only a VersionedUnpickler has - # the BUILD instruction replaced with one that uses _file, and it - # should have _file defined. - # self.dispatch[BUILD[0]] = load_build_with_meta_data - - ########################################################################## - # 'Unpickler' interface - ########################################################################## - - ### public interface ##################################################### - - def find_class(self, module, name): - """Returns the class definition for the named class within the - specified module. - - Overridden here to: - - - Allow updaters to redirect to a different class, possibly - within a different module. - - Ensure that any setstate hooks for the class are called - when the instance of this class is unpickled. - """ - # Remove any extraneous characters that an Unpickler might handle - # but a user wouldn't have included in their mapping definitions. - module = module.strip() - name = name.strip() - - # Attempt to find the class, this may cause a new mapping for that - # very class to be introduced. That's why we ignore the result. - try: - klass = super(VersionedUnpickler, self).find_class(module, name) - except: - pass - - # Determine the target class that the requested class should be - # mapped to according to our updater. The target class is the one - # at the end of any chain of mappings. - original_module, original_name = module, name - if self.updater is not None and self.updater.has_class_mapping( - module, name - ): - module, name = self._get_target_class(module, name) - if module != original_module or name != original_name: - logger.debug( - "Unpickling [%s.%s] as [%s.%s]", - original_module, - original_name, - module, - name, - ) - - # Retrieve the target class definition - try: - klass = super(VersionedUnpickler, self).find_class(module, name) - except Exception as e: - from apptools.sweet_pickle import UnpicklingError - - logger.debug( - "Traceback when finding class [%s.%s]:" % (module, name), - exc_info=True, - ) - raise UnpicklingError( - "Unable to load class [%s.%s]. " - 'Original exception was, "%s". map:%s' - % (module, name, str(e), self.updater.class_map) - ) - - # Make sure we run the updater's state functions if any are declared - # for the target class. - if self.updater is not None and self._has_state_function( - original_module, original_name - ): - self._add_unpickler(klass, original_module, original_name) - - return klass - - ########################################################################## - # 'VersionedUnpickler' interface - ########################################################################## - - ### public interface ##################################################### - - def modify_state(self, obj, state, module, name): - """Called to update the specified state dictionary, which represents - the class of the specified name within the specified module, to - complete the unpickling of the specified object. - """ - # Remove our setstate hook and associated data to ensure that - # instances unpickled through some other framework don't call us. - # IMPORTANT: Do this first to minimize the time this hook is in place! - self._remove_unpickler(obj.__class__) - - # Determine what class and version we're starting from and going to. - # If there is no version information, then assume version 0. (0 is - # like an unversioned version.) - source_key = self.updater.get_version_attribute(module, name) - source_version = state.get(source_key, 0) - target_key = self.updater.get_version_attribute( - obj.__class__.__module__, obj.__class__.__name__ - ) - target_version = getattr(obj, target_key, 0) - - # Iterate through all the updates to the state by going one version - # at a time. Note that we assume there is exactly one path from our - # starting class and version to our ending class and version. As a - # result, we assume we update a given class to its latest version - # before looking for any class mappings. Note that the version in the - # updater is the version to convert *TO*. - version = source_version - next_version = version + 1 - while True: - - # Iterate through all version updates for the current class. - key = self.updater.get_version_attribute(module, name) - while (module, name, next_version) in self.updater.state_functions: - functions = self.updater.state_functions[ - (module, name, next_version) - ] - for f in functions: - logger.debug( - "Modifying state from [%s.%s (v.%s)] to " - + "[%s.%s (v.%s)] using function %s", - module, - name, - version, - module, - name, - next_version, - f, - ) - state = f(state) - - # Avoid infinite loops due to versions not changing. - new_version = state.get(key, version) - if new_version == version: - new_version = version + 1 - version = new_version - next_version = version + 1 - - # If there is one, move to the next class in the chain. (We - # explicitly keep the version number the same.) - if self.updater.has_class_mapping(module, name): - original_module, original_name = module, name - module, name = self.updater.class_map[(module, name)] - logger.debug( - "Modifying state from [%s.%s (v.%s)] to " - + "[%s.%s (v.%s)]", - original_module, - original_name, - version, - module, - name, - version, - ) - else: - break - - # If one exists, call the final class's setstate method. According to - # standard pickling protocol, this method will apply the state to the - # instance so our state becomes None so that we don't try to apply our - # unfinished state to the object. - fn = getattr(obj, _SETSTATE_NAME, None) - if fn is not None: - fn(state) - result = None - version = getattr(obj, target_key) - else: - result = state - - # Something is wrong if we aren't at our target class and version! - if ( - module != obj.__class__.__module__ - or name != obj.__class__.__name__ - or version != target_version - ): - from apptools.sweet_pickle import UnpicklingError - - raise UnpicklingError( - "Unexpected state! Got " - + "[%s.%s (v.%s)] expected [%s.%s (v.%s)]" - % ( - module, - name, - version, - obj.__class__.__module__, - obj.__class__.__name__, - target_version, - ) - ) - - return result - - ### protected interface ################################################## - - def _add_unpickler(self, klass, module, name): - """Modifies the specified class so that our 'modify_state' method - is called when its next instance is unpickled. - """ - logger.debug("Adding unpickler hook to [%s]", klass) - - # Replace the existing setstate method with ours. - self._backup_setstate(klass) - m = __replacement_setstate__.__get__(None, klass) - setattr(klass, _SETSTATE_NAME, m) - - # Add the information necessary to allow this unpickler to run - setattr(klass, _UNPICKLER_DATA, (self, module, name)) - - def _backup_setstate(self, klass): - """Backs up the specified class's setstate method.""" - # We only need to back it up if it actually exists. - method = getattr(klass, _SETSTATE_NAME, None) - if method is not None: - logger.debug( - "Backing up method [%s] to [%s] on [%s]", - _SETSTATE_NAME, - _BACKUP_NAME, - klass, - ) - m = method.__get__(None, klass) - setattr(klass, _BACKUP_NAME, m) - - def _get_target_class(self, module, name): - """Returns the class info that the class, within the specified module - and with the specified name, should be instantiated as according to - our associated updater. - - This is done in a manner that allows for chaining of class mappings - but is tolerant of the fact that a mapping away from an - intermediate class may not be registered until an attempt is made - to load that class. - """ - # Keep a record of the original class asked for. - original_module, original_name = module, name - - # Iterate through any mappings in a manner that allows us to detect any - # infinite loops. - visited = [] - while self.updater.has_class_mapping(module, name): - if (module, name) in visited: - from apptools.sweet_pickle import UnpicklingError - - raise UnpicklingError( - "Detected infinite loop in class " - + "mapping from [%s.%s] to [%s.%s] within Updater [%s]" - % ( - original_module, - original_name, - module, - name, - self.updater, - ) - ) - visited.append((module, name)) - - # Get the mapping for the current class and try loading the class - # to ensure any mappings away from it are registered. - module, name = self.updater.class_map[(module, name)] - try: - super(VersionedUnpickler, self).find_class(module, name) - except: - logger.exception( - "_get_target_class can't find: %s" % (module, name) - ) - pass - - return module, name - - def _has_state_function(self, module, name): - """Returns True if the updater contains any state functions that could - be called by unpickling an instance of the class identified by the - specified module and name. - - Note: If we had a version number we could tell for sure, but we - don't have one so we'll have to settle for 'could' be called. - """ - result = False - - # Iterate through all the class mappings the requested class would - # go through. If any of them have a state function, then we've - # determined our answer and can stop searching. - # - # Note we don't need to check for infinite loops because we're only - # ever called after '_get_target_class' which detects the infinite - # loops. - while not result: - result = self.updater.has_state_function(module, name) - if not result: - if self.updater.has_class_mapping(module, name): - module, name = self.updater.class_map[(module, name)] - else: - break - - return result - - def _remove_unpickler(self, klass): - """Restores the specified class to its unmodified state. Meaning - we won't get called when its next instance is unpickled. - """ - logger.debug("Removing unpickler hook from [%s]", klass) - - # Restore the backed up setstate method - self._restore_setstate(klass) - - # Remove the unpickling data attached to the class. This ensures we - # don't pollute the 'real' attributes of the class. - delattr(klass, _UNPICKLER_DATA) - - def _restore_setstate(self, klass): - """Restores the original setstate method back to its rightful place.""" - # We only need to restore if the backup actually exists. - method = getattr(klass, _BACKUP_NAME, None) - if method is not None: - logger.debug( - "Restoring method [%s] to [%s] on [%s]", - _BACKUP_NAME, - _SETSTATE_NAME, - klass, - ) - delattr(klass, _BACKUP_NAME) - m = method.__get__(None, klass) - setattr(klass, _SETSTATE_NAME, m) - - # Otherwise, we simply remove our setstate. - else: - delattr(klass, _SETSTATE_NAME) diff --git a/setup.cfg b/setup.cfg index acf687653..47d995fa3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,5 @@ per-file-ignores = */api.py:F401, */__init__.py:F401 exclude = # The following list should eventually be empty. apptools/naming/* - apptools/sweet_pickle/* etstool.py examples/* From c3d8f778ccbd30489976c5ec2960ec98641f58f9 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 24 Nov 2020 12:54:10 -0700 Subject: [PATCH 12/14] flake8 --- .../persistence/tests/test_state_function.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/apptools/persistence/tests/test_state_function.py b/apptools/persistence/tests/test_state_function.py index dee6a38bc..cc8458dbb 100644 --- a/apptools/persistence/tests/test_state_function.py +++ b/apptools/persistence/tests/test_state_function.py @@ -19,29 +19,14 @@ import logging # Enthought library imports +from apptools.persistence.tests.state_function_classes import Foo, Bar, Baz from apptools.persistence.versioned_unpickler import VersionedUnpickler from apptools.persistence.updater import Updater -from traits.api import Bool, Float, HasTraits, Int, Str - logger = logging.getLogger(__name__) -############################################################################## -# Classes to use within the tests -############################################################################## - -# Need complete package name so that mapping matches correctly. -# The problem here is the Python loader that will load the same module with -# multiple names in sys.modules due to relative naming. Nice. -from apptools.persistence.tests.state_function_classes import Foo, Bar, Baz - -############################################################################## -# State functions to use within the tests -############################################################################## - - def bar_state_function(self, state): for old, new in [("b1", "b2"), ("f1", "f2"), ("i1", "i2"), ("s1", "s2")]: state[new] = state[old] @@ -61,7 +46,6 @@ def __init__(self): self.setstates = {} - ############################################################################## # class 'StateFunctionTestCase' ############################################################################## @@ -87,7 +71,6 @@ def setUp(self): self.updater = TestUpdater() - ########################################################################## # 'StateFunctionTestCase' interface ########################################################################## From 91968f209a0b2f9e889536a1a15f0437e8ae7c0e Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Tue, 24 Nov 2020 13:23:00 -0700 Subject: [PATCH 13/14] remove/update any references to sweet_pickle in the comments --- apptools/persistence/tests/test_class_mapping.py | 4 ++-- apptools/persistence/tests/test_state_function.py | 4 ++-- apptools/persistence/versioned_unpickler.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apptools/persistence/tests/test_class_mapping.py b/apptools/persistence/tests/test_class_mapping.py index 762992bbc..b033df286 100644 --- a/apptools/persistence/tests/test_class_mapping.py +++ b/apptools/persistence/tests/test_class_mapping.py @@ -44,8 +44,8 @@ class Baz: class ClassMappingTestCase(unittest.TestCase): - """Tests the class mapping functionality of the apptools.sweet_pickle - framework. + """Originally tests for the class mapping functionality of the now deleted + apptools.sweet_pickle framework, converted to use apptools.persistence. """ ########################################################################## diff --git a/apptools/persistence/tests/test_state_function.py b/apptools/persistence/tests/test_state_function.py index cc8458dbb..a7e318421 100644 --- a/apptools/persistence/tests/test_state_function.py +++ b/apptools/persistence/tests/test_state_function.py @@ -52,8 +52,8 @@ def __init__(self): class StateFunctionTestCase(unittest.TestCase): - """Tests the state function functionality of the apptools.sweet_pickle - framework. + """Originally tests for the state function functionality of the now deleted + apptools.sweet_pickle framework, converted to use apptools.persistence. """ ########################################################################## diff --git a/apptools/persistence/versioned_unpickler.py b/apptools/persistence/versioned_unpickler.py index 52cd5b28f..a346c2c95 100644 --- a/apptools/persistence/versioned_unpickler.py +++ b/apptools/persistence/versioned_unpickler.py @@ -82,9 +82,9 @@ def initialize(self, max_pass): generators.remove((o, g)) # Make this a class method since dispatch is a class variable. - # Otherwise, supposing the initial sweet_pickle.load call (which would - # have overloaded the load_build method) makes a pickle.load call at some - # point, we would have the dispatch still pointing to + # Otherwise, supposing the initial VersionedUnpickler.load call (which + # would have overloaded the load_build method) makes a pickle.load call at + # some point, we would have the dispatch still pointing to # NewPickler.load_build whereas the object being passed in will be an # Unpickler instance, causing a TypeError. def load_build(cls, obj): From 9d09a051403e23c11b638d9a6609ea868efdd539 Mon Sep 17 00:00:00 2001 From: Aaron Ayres Date: Wed, 25 Nov 2020 07:13:06 -0700 Subject: [PATCH 14/14] apply suggestions from code review --- .../persistence/tests/test_state_function.py | 8 -------- .../tests/test_two_stage_unpickler.py | 19 ++++++++++--------- docs/releases/upcoming/199.removal.rst | 5 ++++- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/apptools/persistence/tests/test_state_function.py b/apptools/persistence/tests/test_state_function.py index a7e318421..9fdbb74f6 100644 --- a/apptools/persistence/tests/test_state_function.py +++ b/apptools/persistence/tests/test_state_function.py @@ -27,14 +27,6 @@ logger = logging.getLogger(__name__) -def bar_state_function(self, state): - for old, new in [("b1", "b2"), ("f1", "f2"), ("i1", "i2"), ("s1", "s2")]: - state[new] = state[old] - del state[old] - state["_enthought_pickle_version"] = 2 - return state - - class TestUpdater(Updater): def __init__(self): self.refactorings = { diff --git a/apptools/persistence/tests/test_two_stage_unpickler.py b/apptools/persistence/tests/test_two_stage_unpickler.py index 2b7d202d3..777dbf9b7 100644 --- a/apptools/persistence/tests/test_two_stage_unpickler.py +++ b/apptools/persistence/tests/test_two_stage_unpickler.py @@ -81,11 +81,12 @@ def test_generic(self): # This will fail, even though we have a __setstate__ method. s = pickle.dumps(a) new_a = pickle.loads(s) - try: - new_a.x + # Accessing new_a.x is okay + new_a.x + # Accessing y directly would fail + with self.assertRaisesRegex( + AttributeError, "'B' object has no attribute 'y'"): new_a.b_ref.y - except Exception: - pass # This will work! s = pickle.dumps(a) @@ -152,7 +153,7 @@ def __init__(self): self.finder = StringFinder(self.reader, "e") def get(self): - pass + return (self.finder.data, self.reader.data) class ToyAppTestCase(unittest.TestCase): @@ -162,11 +163,11 @@ def test_toy_app(self): a.get() s = pickle.dumps(a) b = pickle.loads(s) - # Won't work. - try: + + with self.assertRaisesRegex( + AttributeError, + "'StringFinder' object has no attribute 'data'"): b.get() - except Exception: - pass # Works fine. c = VersionedUnpickler(io.BytesIO(s)).load() diff --git a/docs/releases/upcoming/199.removal.rst b/docs/releases/upcoming/199.removal.rst index c22d99e8c..b556dc7bf 100644 --- a/docs/releases/upcoming/199.removal.rst +++ b/docs/releases/upcoming/199.removal.rst @@ -1 +1,4 @@ -remove the ``apptools.sweet_pickle`` subpackage. Note that users of sweet_pickle can transition to using ``apptools.persistance`` and pickle from the python standard library (see changes made in this PR to ``apptools.naming`` for more info) (#199) \ No newline at end of file +remove the ``apptools.sweet_pickle`` subpackage. Note that users of +sweet_pickle can in some cases transition to using ``apptools.persistence`` and +pickle from the python standard library (see changes made in this PR to +``apptools.naming`` for more info) (#199) \ No newline at end of file