From c69903283df3355805be97811167c965fbd3165c Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Dec 2019 17:16:34 -0700 Subject: [PATCH 01/13] Add ConfigBlock.inherit_from() method --- pyutilib/misc/config.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index 3bd1bd7c..a5f24925 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -942,6 +942,20 @@ def declare(self, name, config): self._declared.add(name) return ans + def inherit_from(self, other, skip=None): + if not isinstance(other, ConfigBlock): + raise ValueError( + "ConfigBlock.inherit_from() only accepts other ConfigBlocks") + # Note that we duplicate ["other()"] other so that this + # ConfigBlock's entries are independent of the other's + for key in other.iterkeys(): + if skip and key in skip: + continue + if key in self: + raise ValueError("ConfigBlock.inherit_from passed a block " + "with a duplicate field, %s" % (key,)) + self.declare(key, other._data[key]()) + def add(self, name, config): if not self._implicit_declaration: raise ValueError("Key '%s' not defined in Config Block '%s'" From 18cb601d1b0e05d40307ea8fc344028efdac2085 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Dec 2019 17:16:59 -0700 Subject: [PATCH 02/13] Add a skip_implicit option for ConfigBlock.set_value() --- pyutilib/misc/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index a5f24925..f55dfbe9 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -978,7 +978,7 @@ def value(self, accessValue=True): return dict((name, config.value(accessValue)) for name, config in six.iteritems(self._data)) - def set_value(self, value): + def set_value(self, value, skip_implicit=False): if value is None: return self if (type(value) is not dict) and \ @@ -1001,7 +1001,9 @@ def set_value(self, value): if _key in self._data: _decl_map[str(_key)] = key else: - if self._implicit_declaration: + if skip_implicit: + pass + elif self._implicit_declaration: _implicit.append(key) else: raise ValueError( From 7d4019be6997100981345776b92c2762c32a88f1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 25 Feb 2020 07:56:47 -0700 Subject: [PATCH 03/13] set_value should use __setitem__ --- pyutilib/misc/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index f55dfbe9..00d76021 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -1022,7 +1022,7 @@ def set_value(self, value, skip_implicit=False): for key in self._decl_order: if key in _decl_map: #print "Setting", key, " = ", value - self._data[key].set_value(value[_decl_map[key]]) + self[key] = value[_decl_map[key]] # implicit data is declared at the end (in sorted order) for key in sorted(_implicit): self.add(key, value[key]) From bae1bb2c41ec27c28d6957a5934e060b236c8014 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 12 Mar 2020 08:59:22 -0600 Subject: [PATCH 04/13] Adding 'visibility' flag to display() --- pyutilib/misc/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index f55dfbe9..39ba4bdf 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -445,7 +445,8 @@ def import_argparse(self, parsed_args): del parsed_args.__dict__[_dest] return parsed_args - def display(self, content_filter=None, indent_spacing=2, ostream=None): + def display(self, content_filter=None, indent_spacing=2, ostream=None, + visibility=None): if content_filter not in ConfigBlock.content_filters: raise ValueError("unknown content filter '%s'; valid values are %s" % (content_filter, ConfigBlock.content_filters)) @@ -454,13 +455,12 @@ def display(self, content_filter=None, indent_spacing=2, ostream=None): if ostream is None: ostream=stdout - for level, prefix, value, obj in self._data_collector(0, ""): + for lvl, prefix, value, obj in self._data_collector(0, "", visibility): if content_filter == 'userdata' and not obj._userSet: continue _str = _value2string(prefix, value, obj) - _blocks[level:] = [ - ' ' * indent_spacing * level + _str + "\n",] + _blocks[lvl:] = [' ' * indent_spacing * lvl + _str + "\n",] for i, v in enumerate(_blocks): if v is not None: From eb842d46349d18b742ca841d6018e8653cf4fecc Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 12 Mar 2020 08:59:45 -0600 Subject: [PATCH 05/13] Exposing ConfigDict alias to importers --- pyutilib/misc/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index 39ba4bdf..3ad94e8e 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -37,7 +37,7 @@ def dump(x, **args): except ImportError: import __builtin__ as _builtins -__all__ = ('ConfigBlock', 'ConfigList', 'ConfigValue') +__all__ = ('ConfigDict', 'ConfigBlock', 'ConfigList', 'ConfigValue') logger = logging.getLogger('pyutilib.misc.config') From 49aea80173915a74e4f5e9b19f6ea570eb868819 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 12 Mar 2020 09:00:15 -0600 Subject: [PATCH 06/13] Renaming ConfigBlock 'inherit_from' for 'declare_from' --- pyutilib/misc/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index 3ad94e8e..1b99e71b 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -942,17 +942,17 @@ def declare(self, name, config): self._declared.add(name) return ans - def inherit_from(self, other, skip=None): + def declare_from(self, other, skip=None): if not isinstance(other, ConfigBlock): raise ValueError( - "ConfigBlock.inherit_from() only accepts other ConfigBlocks") + "ConfigBlock.declare_from() only accepts other ConfigBlocks") # Note that we duplicate ["other()"] other so that this # ConfigBlock's entries are independent of the other's for key in other.iterkeys(): if skip and key in skip: continue if key in self: - raise ValueError("ConfigBlock.inherit_from passed a block " + raise ValueError("ConfigBlock.declare_from passed a block " "with a duplicate field, %s" % (key,)) self.declare(key, other._data[key]()) From bf4046cadecd535756767fdc63e7bdc2ff2cd4f2 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 12 Mar 2020 09:00:58 -0600 Subject: [PATCH 07/13] Adding initial ConfigBlock documentation --- pyutilib/misc/config.py | 241 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 236 insertions(+), 5 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index 1b99e71b..1ecc0d9d 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -51,6 +51,165 @@ def _munge_name(name, space_to_dash=True): _leadingSpace = re.compile('^([ \n\t]*)') +"""The PyUtilib Config System + +The PyUtilib config system provides a set of three classes (ConfigDict, +ConfigList, and ConfigValue) for managing and documenting structured +configuration information and user input. The system is based around +the ConfigValue class, which provides storage for a single configuration +entry. ConfigValue objects can be grouped using two containers +(ConfigDict and ConfigList), which provide functionality analogous to +Python's dict and list classes, respectively. + +At its simplest, the Config system allows for developers to specify a +dictionary of documented configuration entries, allow users to provide +values for those entries, and retrieve the current values: + +.. doctest:: + :hide: + >>> from pyutilib.misc.config import ( + ... ConfigBlock, ConfigList, ConfigValue, In, + ... ) + +.. doctest:: + >>> config = ConfigBlock() + >>> config.declare('filename', ConfigValue( + ... default=None, + ... domain=str, + ... description="Input file name", + ... )) + >>> config.declare("bound tolerance", ConfigValue( + ... default=1E-5, + ... domain=float, + ... description="Bound tolerance", + ... doc="Relative tolerance for bound feasibility checks" + ... )) + >>> config.declare("iteration limit", ConfigValue( + ... default=30, + ... domain=int, + ... description="Iteration limit", + ... doc="Number of maximum iterations in the decomposition methods" + ... )) + >>> config['filename'] = 'tmp.txt' + >>> print(config['filename']) + tmp.txt + >>> print(config['iteration limit']) + 30 + +For convenience, ConfigBlock objects support read/write access via +attributes (with spaces in the declaration names replaced by +underscores): + +.. doctest:: + >>> print(config.filename) + tmp.txt + >>> print(config.iteration_limit) + 30 + >>> config.iteration_limit = 20 + >>> print(config.iteration_limit) + 20 + +All Config objects support a `domain` keyword that accepts a callable +object (type, function, or callable instance). The domain callable +should take data and map it onto the desired domain, optionally +performing domain validation (see :py:class:`ConfigValue`, +:py:class:`ConfigBlock`, and :py:class:`ConfigList` for more +information). This allows client code to accept a very flexible set of +inputs without "cluttering" the code with input validation: + +.. doctest:: + >>> config.iteration_limit = 35.5 + >>> print(config.iteration_limit) + 35 + >>> print(type(config.iteration_limit).__name__) + int + + +A feature of the Config system is that the core classes all implement +`__call__`, and can themselves be used as `domain` values. Beyond +providing domain verification for complex hierarchical structures, this +feature allows ConfigBlocks to cleanly support the configuration of +derived objects. Consider the following example: + +.. doctest:: + >>> class Base(object): + ... CONFIG = ConfigBlock() + ... CONFIG.declare('filename', ConfigValue( + ... default='input.txt', + ... domain=str, + ... )) + ... def __init__(self, **kwds): + ... c = self.CONFIG(kwds) + ... c.display() + ... + >>> class Derived(Base): + ... CONFIG = Base.CONFIG() + ... CONFIG.declare('pattern', ConfigValue( + ... default=None, + ... domain=str, + ... )) + ... + >>> Base(filename='foo.txt') + filename: foo.txt + >>> Derived(pattern='.*warning') + filename: input.txt + pattern: .*warning + +Here, the base class `Base` declares a class-level attribute CONFIG as a +ConfigBlock containing a single entry (`filename`). The derived class +(`Derived`) then starts by making a copy of the base class' `CONFIG`, +and then defines an additional entry (`pattern`). Instances of the base +class will still create `c` instances that only have the single +`filename` entry, whereas instances of the derived class will have `c` +instances with two entries: the `pattern` entry declared by the derived +class, and the `filename` entry "inherited" from the base class. + +An extension of this design pattern provides a clean approach for +handling "ephemeral" instance options. Consider an interface to an +external "solver". Our class implements a `solve()` method that takes a +problem and sends it to the solver along with some solver configuration +options. We would like to be able to set those options "persistently" +on instances of the interface class, but still override them +"temporarily" for individual calls to `solve()`. We implement this by +creating copies of the class's configuration for both specific instances +and for use by each `solve()` call: + +.. doctest:: + >>> class Solver(object): + ... CONFIG = ConfigBlock() + ... CONFIG.declare('iterlim', ConfigValue( + ... default=10, + ... domain=int, + ... )) + ... def __init__(self, **kwds): + ... self.config = self.CONFIG(kwds) + ... def solve(self, model, **options): + ... config = self.config(options) + ... # Solve the model with the specified iterlim + ... config.display() + ... + >>> solver = Solver() + >>> solver.solve(None) + iterlim: 10 + >>> solver.config.iterlim = 20 + >>> solver.solve(None) + iterlim: 20 + >>> solver.solve(None, iterlim=50) + iterlim: 50 + >>> solver.solve(None) + iterlim: 20 + + +In addition to basic storage and retrieval, the Config system provides +hooks to the argparse command-line argument parsing system. Individual +Config entries can be declared as argparse arguments. To make +declaration simpler, the `declare` method returns the declared Config +object so that the argument declaration can be done inline: + + + +""" + def _strip_indentation(doc): if not doc: @@ -332,11 +491,14 @@ def reset(self): def declare_as_argument(self, *args, **kwds): """Map this Config item to an argparse argument. -Valid arguments include all valid arguments to argparse's -ArgumentParser.add_argument() with the exception of 'default'. In addition, -you may provide a group keyword argument can be used to either pass in a -pre-defined option group or subparser, or else pass in the title of a -group, subparser, or (subparser, group).""" + Valid arguments include all valid arguments to argparse's + ArgumentParser.add_argument() with the exception of 'default'. + In addition, you may provide a group keyword argument can be + used to either pass in a pre-defined option group or subparser, + or else pass in the title of a group, subparser, or (subparser, + group). + + """ if 'default' in kwds: raise TypeError( @@ -625,6 +787,35 @@ def unused_user_values(self): class ConfigValue(ConfigBase): + """Store and manipulate a single configuration value. + + Parameters + ---------- + default: optional + The default value that this ConfigValue will take if no value is + provided. + domain: callable, optional + The domain can be any callable that accepts a candidate value + and returns the value converted to the desired type, optionally + performing any data validation. Examples include type + constructors like `int` or `float`. More complex domain + examples include callable objects; for example, the + :py:class:`In` class that ensures that the value falls into an + acceptable set. + description: str, optional + The short description of this value + doc: str, optional + The long documentation string for this value + visibility: int, optional + The visibility of this ConfigValue when generating templates and + documentation. Visibility supports specification of "advanced" + or "developer" options. ConfigValues with visibility=0 (the + default) will always be printed / included. ConfigValues + with higher visibility values will only be included when the + generation method specifies a visibility greater than or equal + to the visibility of this object. + + """ def __init__(self, *args, **kwds): ConfigBase.__init__(self, *args, **kwds) @@ -1070,3 +1261,43 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False): ConfigBlock.keys = ConfigBlock.iterkeys ConfigBlock.values = ConfigBlock.itervalues ConfigBlock.items = ConfigBlock.iteritems + + +class In(object): + """A ConfigValue domain validator that checks values against a set + + Instances of In map incoming values to the desired type (if domain + is specified) and check that the resulting value is in the specified + set. + + Examples + -------- + >>> c = ConfigValue(domain=In(['foo', 'bar', '0'], domain=str)) + >>> c.set_value('foo') + >>> c.display + foo + >>> c.set_value(3) + ValueError: invalid value for configuration '': + Failed casting 3 + to + Error: value 3 not in domain ['foo', 'bar'] + >>> c.display + foo + >>> c.set_value(0) + >>> c.display + '0' + + """ + + def __init__(self, allowable, domain=None): + self._allowable = allowable + self._domain = domain + + def __call__(self, value): + if self._domain is not None: + v = self._domain(value) + else: + v = value + if v in self._allowable: + return v + raise ValueError("value %s not in domain %s" % (value, self._allowable)) From cfb15bae3a965dca18a351d4840d87e5078dad05 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 17 Mar 2020 22:55:15 -0600 Subject: [PATCH 08/13] Adding argparse documentation; moving so python picks up the module docstring --- pyutilib/misc/config.py | 162 ++++++++++++++++++++++++++++------------ 1 file changed, 113 insertions(+), 49 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index e20eb0a3..bf3f3b1a 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -7,55 +7,7 @@ # the U.S. Government retains certain rights in this software. # _________________________________________________________________________ -import re -from sys import exc_info, stdout -from textwrap import wrap -import logging -import pickle - -import six -from six.moves import xrange - -def _dump(*args, **kwds): - try: - from yaml import dump - except ImportError: - #dump = lambda x,**y: str(x) - # YAML uses lowercase True/False - def dump(x, **args): - if type(x) is bool: - return str(x).lower() - return str(x) - assert '_dump' in globals() - globals()['_dump'] = dump - return dump(*args, **kwds) - -try: - import argparse - argparse_is_available = True -except ImportError: - argparse_is_available = False - -try: - import builtins as _builtins -except ImportError: - import __builtin__ as _builtins - -__all__ = ('ConfigDict', 'ConfigBlock', 'ConfigList', 'ConfigValue') - -logger = logging.getLogger('pyutilib.misc.config') - - -def _munge_name(name, space_to_dash=True): - if space_to_dash: - name = re.sub(r'\s', '-', name) - name = re.sub(r'_', '-', name) - return re.sub(r'[^a-zA-Z0-9-_]', '_', name) - - -_leadingSpace = re.compile('^([ \n\t]*)') - -"""The PyUtilib Config System +"""The PyUtilib Configuration System The PyUtilib config system provides a set of three classes (ConfigDict, ConfigList, and ConfigValue) for managing and documenting structured @@ -71,6 +23,7 @@ def _munge_name(name, space_to_dash=True): .. doctest:: :hide: + >>> import argparse >>> from pyutilib.misc.config import ( ... ConfigBlock, ConfigList, ConfigValue, In, ... ) @@ -210,10 +163,121 @@ class will still create `c` instances that only have the single declaration simpler, the `declare` method returns the declared Config object so that the argument declaration can be done inline: +.. doctest:: + >>> config = ConfigBlock() + >>> config.declare('iterlim', ConfigValue( + ... domain=int, + ... default=100, + ... description="iteration limit", + ... )).declare_as_argument() + >>> config.declare('lbfgs', ConfigValue( + ... domain=bool, + ... description="use limited memory BFGS update", + ... )).declare_as_argument() + >>> config.declare('linesearch', ConfigValue( + ... domain=bool, + ... default=True, + ... description="use line search", + ... )).declare_as_argument() + >>> config.declare('relative tolerance', ConfigValue( + ... domain=float, + ... description="relative convergence tolerance", + ... )).declare_as_argument('--reltol', '-r', group='Tolerances') + >>> config.declare('absolute tolerance', ConfigValue( + ... domain=float, + ... description="absolute convergence tolerance", + ... )).declare_as_argument('--abstol', '-a', group='Tolerances') + +The ConfigBlock can then be used to initialize (or augment) an argparse +ArgumentParser object: + +.. doctest:: + >>> parser = argparse.ArgumentParser("tester") + >>> config.initialize_argparse(parser) + + +Key information from the ConfigBlock is automatically transferred over +to the ArgumentParser object: + +.. doctest:: + >>> print(parser.format_help()) + usage: tester [-h] [--iterlim INT] [--lbfgs] [--disable-linesearch] + [--reltol FLOAT] [--abstol FLOAT] + + optional arguments: + -h, --help show this help message and exit + --iterlim INT iteration limit + --lbfgs use limited memory BFGS update + --disable-linesearch [DON'T] use line search + + Tolerances: + --reltol FLOAT, -r FLOAT + relative convergence tolerance + --abstol FLOAT, -a FLOAT + absolute convergence tolerance + +Parsed arguments can then be imported back into the ConfigBlock: + +.. doctest:: + >>> args=parser.parse_args(['--lbfgs', '--reltol', '0.1', '-a', '0.2']) + >>> config.import_argparse(args) + >>> config.display() + iterlim: 100 + lbfgs: true + linesearch: true + relative tolerance: 0.1 + absolute tolerance: 0.2 """ +import re +from sys import exc_info, stdout +from textwrap import wrap +import logging +import pickle + +import six +from six.moves import xrange + +def _dump(*args, **kwds): + try: + from yaml import dump + except ImportError: + #dump = lambda x,**y: str(x) + # YAML uses lowercase True/False + def dump(x, **args): + if type(x) is bool: + return str(x).lower() + return str(x) + assert '_dump' in globals() + globals()['_dump'] = dump + return dump(*args, **kwds) + +try: + import argparse + argparse_is_available = True +except ImportError: + argparse_is_available = False + +try: + import builtins as _builtins +except ImportError: + import __builtin__ as _builtins + +__all__ = ('ConfigDict', 'ConfigBlock', 'ConfigList', 'ConfigValue') + +logger = logging.getLogger('pyutilib.misc.config') + + +def _munge_name(name, space_to_dash=True): + if space_to_dash: + name = re.sub(r'\s', '-', name) + name = re.sub(r'_', '-', name) + return re.sub(r'[^a-zA-Z0-9-_]', '_', name) + + +_leadingSpace = re.compile('^([ \n\t]*)') def _strip_indentation(doc): if not doc: From 67a125101ce849345d1fa48195e05b513439b391 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Mar 2020 00:32:22 -0600 Subject: [PATCH 09/13] Resolve block_end bug in generate_documentation --- pyutilib/misc/config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index bf3f3b1a..ebbe23e6 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -831,12 +831,13 @@ def generate_documentation\ elif item_end: os.write(indent + item_end) while level: - indent = indent[:-1 * indent_spacing] _last = level.pop() - if '%s' in block_end: - os.write(indent + block_end % _last.name()) - else: - os.write(indent + block_end) + if _last is not None: + indent = indent[:-1 * indent_spacing] + if '%s' in block_end: + os.write(indent + block_end % _last.name()) + else: + os.write(indent + block_end) return os.getvalue() def user_values(self): From a0ce78c06ce30b5de90adab5e56dfd35ba4fc196 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Mar 2020 00:32:30 -0600 Subject: [PATCH 10/13] Adding more Config documentation --- pyutilib/misc/config.py | 233 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 215 insertions(+), 18 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index ebbe23e6..fc06074b 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -25,11 +25,11 @@ :hide: >>> import argparse >>> from pyutilib.misc.config import ( - ... ConfigBlock, ConfigList, ConfigValue, In, + ... ConfigDict, ConfigList, ConfigValue, In, ... ) .. doctest:: - >>> config = ConfigBlock() + >>> config = ConfigDict() >>> config.declare('filename', ConfigValue( ... default=None, ... domain=str, @@ -53,7 +53,7 @@ >>> print(config['iteration limit']) 30 -For convenience, ConfigBlock objects support read/write access via +For convenience, ConfigDict objects support read/write access via attributes (with spaces in the declaration names replaced by underscores): @@ -70,7 +70,7 @@ object (type, function, or callable instance). The domain callable should take data and map it onto the desired domain, optionally performing domain validation (see :py:class:`ConfigValue`, -:py:class:`ConfigBlock`, and :py:class:`ConfigList` for more +:py:class:`ConfigDict`, and :py:class:`ConfigList` for more information). This allows client code to accept a very flexible set of inputs without "cluttering" the code with input validation: @@ -81,16 +81,18 @@ >>> print(type(config.iteration_limit).__name__) int +Configuring class hierarchies +============================= A feature of the Config system is that the core classes all implement `__call__`, and can themselves be used as `domain` values. Beyond providing domain verification for complex hierarchical structures, this -feature allows ConfigBlocks to cleanly support the configuration of +feature allows ConfigDicts to cleanly support the configuration of derived objects. Consider the following example: .. doctest:: >>> class Base(object): - ... CONFIG = ConfigBlock() + ... CONFIG = ConfigDict() ... CONFIG.declare('filename', ConfigValue( ... default='input.txt', ... domain=str, @@ -113,7 +115,7 @@ pattern: .*warning Here, the base class `Base` declares a class-level attribute CONFIG as a -ConfigBlock containing a single entry (`filename`). The derived class +ConfigDict containing a single entry (`filename`). The derived class (`Derived`) then starts by making a copy of the base class' `CONFIG`, and then defines an additional entry (`pattern`). Instances of the base class will still create `c` instances that only have the single @@ -133,7 +135,7 @@ class will still create `c` instances that only have the single .. doctest:: >>> class Solver(object): - ... CONFIG = ConfigBlock() + ... CONFIG = ConfigDict() ... CONFIG.declare('iterlim', ConfigValue( ... default=10, ... domain=int, @@ -157,6 +159,9 @@ class will still create `c` instances that only have the single iterlim: 20 +Interacting with argparse +========================= + In addition to basic storage and retrieval, the Config system provides hooks to the argparse command-line argument parsing system. Individual Config entries can be declared as argparse arguments. To make @@ -164,7 +169,7 @@ class will still create `c` instances that only have the single object so that the argument declaration can be done inline: .. doctest:: - >>> config = ConfigBlock() + >>> config = ConfigDict() >>> config.declare('iterlim', ConfigValue( ... domain=int, ... default=100, @@ -188,8 +193,7 @@ class will still create `c` instances that only have the single ... description="absolute convergence tolerance", ... )).declare_as_argument('--abstol', '-a', group='Tolerances') - -The ConfigBlock can then be used to initialize (or augment) an argparse +The ConfigDict can then be used to initialize (or augment) an argparse ArgumentParser object: .. doctest:: @@ -197,7 +201,7 @@ class will still create `c` instances that only have the single >>> config.initialize_argparse(parser) -Key information from the ConfigBlock is automatically transferred over +Key information from the ConfigDict is automatically transferred over to the ArgumentParser object: .. doctest:: @@ -217,7 +221,7 @@ class will still create `c` instances that only have the single --abstol FLOAT, -a FLOAT absolute convergence tolerance -Parsed arguments can then be imported back into the ConfigBlock: +Parsed arguments can then be imported back into the ConfigDict: .. doctest:: >>> args=parser.parse_args(['--lbfgs', '--reltol', '0.1', '-a', '0.2']) @@ -229,6 +233,126 @@ class will still create `c` instances that only have the single relative tolerance: 0.1 absolute tolerance: 0.2 +Accessing user-specified values +=============================== + +It is frequently useful to know which values a user explicitly set, and +which values a user explicitly set, but have never been retrieved. The +configuration system provides two gemerator methods to return the items +that a user explicitly set (`user_values`) and the items that were set +but never retrieved (`unused_user_values`): + +.. doctest:: + >>> print([val.name() for val in config.user_values()]) + ['lbfgs', 'relative tolerance', 'absolute tolerance'] + >>> print(config.relative_tolerance) + 0.1 + >>> print([val.name() for val in config.unused_user_values()]) + ['lbfgs', 'absolute tolerance'] + +Generating output & documentation +================================= + +Configuration objects support three methods for generating output and +documentation: `display()`, `generate_yaml_template()`, and +`generate_documentation()`. The simplest is `display()`, which prints +out the current values of the configuration object (and if it is a +container type, all of it's children). `generate_yaml_template` is +simular to `display`, but also includes the description fields as +formatted comments. + +.. doctest:: + >>> solver_config = config + >>> config = ConfigDict() + >>> config.declare('output', ConfigValue( + ... default='results.yml', + ... domain=str, + ... description='output results filename' + ... )) + >>> config.declare('verbose', ConfigValue( + ... default=0, + ... domain=int, + ... description='output verbosity', + ... doc='This sets the system verbosity. The default (0) only logs ' + ... 'warnings and errors. Larger integer values will produce ' + ... 'additional log messages.', + ... )) + >>> config.declare('solvers', ConfigList( + ... domain=solver_config, + ... description='list of solvers to apply', + ... )) + >>> config.display() + output: results.yml + verbose: 0 + solvers: [] + >>> print(config.generate_yaml_template()) + output: results.yml # output results filename + verbose: 0 # output verbosity + solvers: [] # list of solvers to apply + +It is important to note that both methods document the current state of +the configuration object. So, in the example above, since the `solvers` +list is empty, you will not get any information on the elements in the +list. Of course, if you add a value to the list, then the data will be +output: + +.. doctest:: + >>> tmp = config() + >>> tmp.solvers.append({}) + >>> tmp.display() + output: results.yml + verbose: 0 + solvers: + - + iterlim: 100 + lbfgs: true + linesearch: true + relative tolerance: 0.1 + absolute tolerance: 0.2 + >>> print(tmp.generate_yaml_template()) + output: results.yml # output results filename + verbose: 0 # output verbosity + solvers: # list of solvers to apply + - + iterlim: 100 # iteration limit + lbfgs: true # use limited memory BFGS update + linesearch: true # use line search + relative tolerance: 0.1 # relative convergence tolerance + absolute tolerance: 0.2 # absolute convergence tolerance + +The third method (:py:meth:`generate_documentation`) behaves +differently. This method is designed to generate reference +documentation. For each configuration item, the `doc` field is output. +If the item has no `doc`, then the `description` field is used. + +List containers have their *domain* documented and not their current +values. The documentation can be configured through optional arguments. +The defaults generate LaTeX documentation: + +.. doctest:: + >>> print(config.generate_documentation()) + \begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em] + \item[{output}]\hfill + \\output results filename + \item[{verbose}]\hfill + \\This sets the system verbosity. The default (0) only logs warnings and + errors. Larger integer values will produce additional log messages. + \item[{solvers}]\hfill + \\list of solvers to apply + \begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em] + \item[{iterlim}]\hfill + \\iteration limit + \item[{lbfgs}]\hfill + \\use limited memory BFGS update + \item[{linesearch}]\hfill + \\use line search + \item[{relative tolerance}]\hfill + \\relative convergence tolerance + \item[{absolute tolerance}]\hfill + \\absolute convergence tolerance + \end{description} + \end{description} + """ import re @@ -863,18 +987,23 @@ class ConfigValue(ConfigBase): default: optional The default value that this ConfigValue will take if no value is provided. + domain: callable, optional The domain can be any callable that accepts a candidate value and returns the value converted to the desired type, optionally - performing any data validation. Examples include type - constructors like `int` or `float`. More complex domain - examples include callable objects; for example, the - :py:class:`In` class that ensures that the value falls into an - acceptable set. + performing any data validation. The result will be stored into + the ConfigValue. Examples include type constructors like `int` + or `float`. More complex domain examples include callable + objects; for example, the :py:class:`In` class that ensures that + the value falls into an acceptable set or even a complete + :py:class:`ConfigDict` instance. + description: str, optional The short description of this value + doc: str, optional The long documentation string for this value + visibility: int, optional The visibility of this ConfigValue when generating templates and documentation. Visibility supports specification of "advanced" @@ -906,6 +1035,43 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False): class ConfigList(ConfigBase): + """Store and manipulate a list of configuration values. + + Parameters + ---------- + default: optional + The default value that this ConfigList will take if no value is + provided. If default is a list or ConfigList, then each member + is cast to the ConfigList's domain to build the default value, + otherwise the default is cast to the domain and forms a default + list with a single element. + + domain: callable, optional + The domain can be any callable that accepts a candidate value + and returns the value converted to the desired type, optionally + performing any data validation. The result will be stored / + added to the ConfigList. Examples include type constructors + like `int` or `float`. More complex domain examples include + callable objects; for example, the :py:class:`In` class that + ensures that the value falls into an acceptable set or even a + complete :py:class:`ConfigDict` instance. + + description: str, optional + The short description of this list + + doc: str, optional + The long documentation string for this list + + visibility: int, optional + The visibility of this ConfigList when generating templates and + documentation. Visibility supports specification of "advanced" + or "developer" options. ConfigLists with visibility=0 (the + default) will always be printed / included. ConfigLists + with higher visibility values will only be included when the + generation method specifies a visibility greater than or equal + to the visibility of this object. + + """ def __init__(self, *args, **kwds): ConfigBase.__init__(self, *args, **kwds) @@ -1044,6 +1210,37 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False): class ConfigBlock(ConfigBase): + """Store and manipulate a dictionary of configuration values. + + Parameters + ---------- + description: str, optional + The short description of this list + + doc: str, optional + The long documentation string for this list + + implicit: bool, optional + If True, the ConfigDict will allow "implicitly" declared + keys, that is, keys can be stored into the ConfigDict that + were not prevously declared using :py:meth:`declare` or + :py:meth:`declare_from`. + + implicit_domain: callable, optional + The domain that will be used for any implicitly-declared keys. + Follows the same rules as :py:meth:`ConfigValue`'s `domain`. + + visibility: int, optional + The visibility of this ConfigDict when generating templates and + documentation. Visibility supports specification of "advanced" + or "developer" options. ConfigDicts with visibility=0 (the + default) will always be printed / included. ConfigDicts + with higher visibility values will only be included when the + generation method specifies a visibility greater than or equal + to the visibility of this object. + + """ + content_filters = (None, 'all', 'userdata') __slots__ = ('_decl_order', '_declared', '_implicit_declaration', From 4b6e1d87b213fe2f92ee2b6dc277acce9afee577 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Mar 2020 01:00:27 -0600 Subject: [PATCH 11/13] Fixing doctests --- pyutilib/misc/config.py | 83 +++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index fc06074b..0b8605d8 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -23,30 +23,35 @@ .. doctest:: :hide: + >>> import argparse >>> from pyutilib.misc.config import ( ... ConfigDict, ConfigList, ConfigValue, In, ... ) .. doctest:: + >>> config = ConfigDict() >>> config.declare('filename', ConfigValue( ... default=None, ... domain=str, ... description="Input file name", ... )) + >>> config.declare("bound tolerance", ConfigValue( ... default=1E-5, ... domain=float, ... description="Bound tolerance", ... doc="Relative tolerance for bound feasibility checks" ... )) + >>> config.declare("iteration limit", ConfigValue( ... default=30, ... domain=int, ... description="Iteration limit", ... doc="Number of maximum iterations in the decomposition methods" ... )) + >>> config['filename'] = 'tmp.txt' >>> print(config['filename']) tmp.txt @@ -58,6 +63,7 @@ underscores): .. doctest:: + >>> print(config.filename) tmp.txt >>> print(config.iteration_limit) @@ -75,6 +81,7 @@ inputs without "cluttering" the code with input validation: .. doctest:: + >>> config.iteration_limit = 35.5 >>> print(config.iteration_limit) 35 @@ -91,6 +98,7 @@ derived objects. Consider the following example: .. doctest:: + >>> class Base(object): ... CONFIG = ConfigDict() ... CONFIG.declare('filename', ConfigValue( @@ -108,9 +116,9 @@ ... domain=str, ... )) ... - >>> Base(filename='foo.txt') + >>> tmp = Base(filename='foo.txt') filename: foo.txt - >>> Derived(pattern='.*warning') + >>> tmp = Derived(pattern='.*warning') filename: input.txt pattern: .*warning @@ -134,6 +142,7 @@ class will still create `c` instances that only have the single and for use by each `solve()` call: .. doctest:: + >>> class Solver(object): ... CONFIG = ConfigDict() ... CONFIG.declare('iterlim', ConfigValue( @@ -169,34 +178,41 @@ class will still create `c` instances that only have the single object so that the argument declaration can be done inline: .. doctest:: + >>> config = ConfigDict() >>> config.declare('iterlim', ConfigValue( ... domain=int, ... default=100, ... description="iteration limit", ... )).declare_as_argument() + >>> config.declare('lbfgs', ConfigValue( ... domain=bool, ... description="use limited memory BFGS update", ... )).declare_as_argument() + >>> config.declare('linesearch', ConfigValue( ... domain=bool, ... default=True, ... description="use line search", ... )).declare_as_argument() + >>> config.declare('relative tolerance', ConfigValue( ... domain=float, ... description="relative convergence tolerance", ... )).declare_as_argument('--reltol', '-r', group='Tolerances') + >>> config.declare('absolute tolerance', ConfigValue( ... domain=float, ... description="absolute convergence tolerance", ... )).declare_as_argument('--abstol', '-a', group='Tolerances') + The ConfigDict can then be used to initialize (or augment) an argparse ArgumentParser object: .. doctest:: + >>> parser = argparse.ArgumentParser("tester") >>> config.initialize_argparse(parser) @@ -205,6 +221,7 @@ class will still create `c` instances that only have the single to the ArgumentParser object: .. doctest:: + >>> print(parser.format_help()) usage: tester [-h] [--iterlim INT] [--lbfgs] [--disable-linesearch] [--reltol FLOAT] [--abstol FLOAT] @@ -220,12 +237,14 @@ class will still create `c` instances that only have the single relative convergence tolerance --abstol FLOAT, -a FLOAT absolute convergence tolerance + Parsed arguments can then be imported back into the ConfigDict: .. doctest:: + >>> args=parser.parse_args(['--lbfgs', '--reltol', '0.1', '-a', '0.2']) - >>> config.import_argparse(args) + >>> args = config.import_argparse(args) >>> config.display() iterlim: 100 lbfgs: true @@ -243,6 +262,7 @@ class will still create `c` instances that only have the single but never retrieved (`unused_user_values`): .. doctest:: + >>> print([val.name() for val in config.user_values()]) ['lbfgs', 'relative tolerance', 'absolute tolerance'] >>> print(config.relative_tolerance) @@ -262,6 +282,7 @@ class will still create `c` instances that only have the single formatted comments. .. doctest:: + >>> solver_config = config >>> config = ConfigDict() >>> config.declare('output', ConfigValue( @@ -269,6 +290,7 @@ class will still create `c` instances that only have the single ... domain=str, ... description='output results filename' ... )) + >>> config.declare('verbose', ConfigValue( ... default=0, ... domain=int, @@ -277,10 +299,12 @@ class will still create `c` instances that only have the single ... 'warnings and errors. Larger integer values will produce ' ... 'additional log messages.', ... )) + >>> config.declare('solvers', ConfigList( ... domain=solver_config, ... description='list of solvers to apply', ... )) + >>> config.display() output: results.yml verbose: 0 @@ -289,6 +313,7 @@ class will still create `c` instances that only have the single output: results.yml # output results filename verbose: 0 # output verbosity solvers: [] # list of solvers to apply + It is important to note that both methods document the current state of the configuration object. So, in the example above, since the `solvers` @@ -297,6 +322,7 @@ class will still create `c` instances that only have the single output: .. doctest:: + >>> tmp = config() >>> tmp.solvers.append({}) >>> tmp.display() @@ -319,6 +345,7 @@ class will still create `c` instances that only have the single linesearch: true # use line search relative tolerance: 0.1 # relative convergence tolerance absolute tolerance: 0.2 # absolute convergence tolerance + The third method (:py:meth:`generate_documentation`) behaves differently. This method is designed to generate reference @@ -330,33 +357,35 @@ class will still create `c` instances that only have the single The defaults generate LaTeX documentation: .. doctest:: + >>> print(config.generate_documentation()) - \begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em] - \item[{output}]\hfill - \\output results filename - \item[{verbose}]\hfill - \\This sets the system verbosity. The default (0) only logs warnings and + \\begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em] + \\item[{output}]\hfill + \\\\output results filename + \\item[{verbose}]\hfill + \\\\This sets the system verbosity. The default (0) only logs warnings and errors. Larger integer values will produce additional log messages. - \item[{solvers}]\hfill - \\list of solvers to apply - \begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em] - \item[{iterlim}]\hfill - \\iteration limit - \item[{lbfgs}]\hfill - \\use limited memory BFGS update - \item[{linesearch}]\hfill - \\use line search - \item[{relative tolerance}]\hfill - \\relative convergence tolerance - \item[{absolute tolerance}]\hfill - \\absolute convergence tolerance - \end{description} - \end{description} + \\item[{solvers}]\hfill + \\\\list of solvers to apply + \\begin{description}[topsep=0pt,parsep=0.5em,itemsep=-0.4em] + \\item[{iterlim}]\hfill + \\\\iteration limit + \\item[{lbfgs}]\hfill + \\\\use limited memory BFGS update + \\item[{linesearch}]\hfill + \\\\use line search + \\item[{relative tolerance}]\hfill + \\\\relative convergence tolerance + \\item[{absolute tolerance}]\hfill + \\\\absolute convergence tolerance + \\end{description} + \\end{description} + """ import re -from sys import exc_info, stdout +import sys from textwrap import wrap import logging import pickle @@ -486,7 +515,7 @@ def _picklable(field,obj): # either: exceeding recursion depth raises a RuntimeError # through 3.4, then switches to a RecursionError (a derivative # of RuntimeError). - if isinstance(exc_info()[0], RuntimeError): + if isinstance(sys.exc_info()[0], RuntimeError): raise _picklable.known[ftype] = False return _UnpickleableDomain(obj) @@ -654,7 +683,7 @@ def _cast(self, value): else: return self._domain() except: - err = exc_info()[1] + err = sys.exc_info()[1] if hasattr(self._domain, '__name__'): _dom = self._domain.__name__ else: @@ -807,7 +836,7 @@ def display(self, content_filter=None, indent_spacing=2, ostream=None, _blocks = [] if ostream is None: - ostream=stdout + ostream=sys.stdout for lvl, prefix, value, obj in self._data_collector(0, "", visibility): if content_filter == 'userdata' and not obj._userSet: From ba22c6a05bc76a077fd46f75608336131a05b4bf Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Mar 2020 01:06:55 -0600 Subject: [PATCH 12/13] Rename ConfigBlock -> ConfigDict --- pyutilib/misc/config.py | 68 +++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index 0b8605d8..be3bc012 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -7,7 +7,10 @@ # the U.S. Government retains certain rights in this software. # _________________________________________________________________________ -"""The PyUtilib Configuration System +""" +================================= +The PyUtilib Configuration System +================================= The PyUtilib config system provides a set of three classes (ConfigDict, ConfigList, and ConfigValue) for managing and documenting structured @@ -604,7 +607,7 @@ def __call__(self, value=NoArgument, default=NoArgument, domain=NoArgument, kwds['visibility'] = ( self._visibility if visibility is ConfigBase.NoArgument else visibility ) - if isinstance(self, ConfigBlock): + if isinstance(self, ConfigDict): kwds['implicit'] = ( self._implicit_declaration if implicit is ConfigBase.NoArgument else implicit ) @@ -614,10 +617,10 @@ def __call__(self, value=NoArgument, default=NoArgument, domain=NoArgument, implicit_domain ) if domain is not ConfigBase.NoArgument: logger.warn("domain ignored by __call__(): " - "class is a ConfigBlock" % (type(self),)) + "class is a ConfigDict" % (type(self),)) if default is not ConfigBase.NoArgument: logger.warn("default ignored by __call__(): " - "class is a ConfigBlock" % (type(self),)) + "class is a ConfigDict" % (type(self),)) else: kwds['default'] = ( self.value() if default is ConfigBase.NoArgument else @@ -627,15 +630,15 @@ def __call__(self, value=NoArgument, default=NoArgument, domain=NoArgument, domain ) if implicit is not ConfigBase.NoArgument: logger.warn("implicit ignored by __call__(): " - "class %s is not a ConfigBlock" % (type(self),)) + "class %s is not a ConfigDict" % (type(self),)) if implicit_domain is not ConfigBase.NoArgument: logger.warn("implicit_domain ignored by __call__(): " - "class %s is not a ConfigBlock" % (type(self),)) + "class %s is not a ConfigDict" % (type(self),)) # Copy over any other object-specific information (mostly Block # definitions) ans = self.__class__(**kwds) - if isinstance(self, ConfigBlock): + if isinstance(self, ConfigDict): for k in self._decl_order: if preserve_implicit or k in self._declared: v = self._data[k] @@ -830,9 +833,9 @@ def import_argparse(self, parsed_args): def display(self, content_filter=None, indent_spacing=2, ostream=None, visibility=None): - if content_filter not in ConfigBlock.content_filters: + if content_filter not in ConfigDict.content_filters: raise ValueError("unknown content filter '%s'; valid values are %s" - % (content_filter, ConfigBlock.content_filters)) + % (content_filter, ConfigDict.content_filters)) _blocks = [] if ostream is None: @@ -1127,7 +1130,7 @@ def __getitem__(self, key): return val def get(self, key, default=ConfigBase.NoArgument): - # Note: get() is borrowed from ConfigBlock for cases where we + # Note: get() is borrowed from ConfigDict for cases where we # want the raw stored object (and to aviod the implicit # conversion of ConfigValue members to their stored data). try: @@ -1187,7 +1190,7 @@ def reset(self): # entries will get their userSet flag set. This is wrong, as # reset() should conceptually reset teh object to it's default # state (e.g., before the user ever had a chance to mess with - # things). As the list could contain a ConfigBlock, this is a + # things). As the list could contain a ConfigDict, this is a # recursive operation to put the userSet values back. for val in self.user_values(): val._userSet = False @@ -1238,7 +1241,7 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False): yield v -class ConfigBlock(ConfigBase): +class ConfigDict(ConfigBase): """Store and manipulate a dictionary of configuration values. Parameters @@ -1293,13 +1296,13 @@ def __init__(self, self._data = {} def __getstate__(self): - state = super(ConfigBlock, self).__getstate__() - state.update((key, getattr(self, key)) for key in ConfigBlock.__slots__) + state = super(ConfigDict, self).__getstate__() + state.update((key, getattr(self, key)) for key in ConfigDict.__slots__) state['_implicit_domain'] = _picklable(state['_implicit_domain'], self) return state def __setstate__(self, state): - state = super(ConfigBlock, self).__setstate__(state) + state = super(ConfigDict, self).__setstate__(state) for x in six.itervalues(self._data): x._parent = self @@ -1343,7 +1346,7 @@ def __setitem__(self, key, val): def __delitem__(self, key): # Note that this will produce a KeyError if the key is not valid - # for this ConfigBlock. + # for this ConfigDict. del self._data[key] # Clean up the other data structures self._decl_order.remove(key) @@ -1363,22 +1366,22 @@ def __getattr__(self, name): # Note: __getattr__ is only called after all "usual" attribute # lookup methods have failed. So, if we get here, we already # know that key is not a __slot__ or a method, etc... - #if name in ConfigBlock._all_slots: - # return super(ConfigBlock,self).__getattribute__(name) + #if name in ConfigDict._all_slots: + # return super(ConfigDict,self).__getattribute__(name) if name not in self._data: _name = name.replace('_', ' ') if _name not in self._data: raise AttributeError("Unknown attribute '%s'" % name) name = _name - return ConfigBlock.__getitem__(self, name) + return ConfigDict.__getitem__(self, name) def __setattr__(self, name, value): - if name in ConfigBlock._all_slots: - super(ConfigBlock, self).__setattr__(name, value) + if name in ConfigDict._all_slots: + super(ConfigDict, self).__setattr__(name, value) else: if name not in self._data: name = name.replace('_', ' ') - ConfigBlock.__setitem__(self, name, value) + ConfigDict.__setitem__(self, name, value) def iterkeys(self): return self._decl_order.__iter__() @@ -1429,16 +1432,16 @@ def declare(self, name, config): return ans def declare_from(self, other, skip=None): - if not isinstance(other, ConfigBlock): + if not isinstance(other, ConfigDict): raise ValueError( - "ConfigBlock.declare_from() only accepts other ConfigBlocks") + "ConfigDict.declare_from() only accepts other ConfigDicts") # Note that we duplicate ["other()"] other so that this - # ConfigBlock's entries are independent of the other's + # ConfigDict's entries are independent of the other's for key in other.iterkeys(): if skip and key in skip: continue if key in self: - raise ValueError("ConfigBlock.declare_from passed a block " + raise ValueError("ConfigDict.declare_from passed a block " "with a duplicate field, %s" % (key,)) self.declare(key, other._data[key]()) @@ -1468,7 +1471,7 @@ def set_value(self, value, skip_implicit=False): if value is None: return self if (type(value) is not dict) and \ - (not isinstance(value, ConfigBlock)): + (not isinstance(value, ConfigDict)): raise ValueError("Expected dict value for %s.set_value, found %s" % (self.name(True), type(value).__name__)) if not value: @@ -1546,16 +1549,15 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False): visibility, docMode): yield v -# Future-proofing: We will be renaming the ConfigBlock to ConfigDict in -# the future -ConfigDict = ConfigBlock +# Backwards compatibility: ConfigDick was originally named ConfigBlock. +ConfigBlock = ConfigDict # In Python3, the items(), etc methods of dict-like things return # generator-like objects. if six.PY3: - ConfigBlock.keys = ConfigBlock.iterkeys - ConfigBlock.values = ConfigBlock.itervalues - ConfigBlock.items = ConfigBlock.iteritems + ConfigDict.keys = ConfigDict.iterkeys + ConfigDict.values = ConfigDict.itervalues + ConfigDict.items = ConfigDict.iteritems class In(object): From f129c2e22742e5c4501b13e9cda093fdd70b062b Mon Sep 17 00:00:00 2001 From: John Siirola Date: Wed, 18 Mar 2020 01:27:56 -0600 Subject: [PATCH 13/13] Docstring formatting updates --- pyutilib/misc/config.py | 50 +++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/pyutilib/misc/config.py b/pyutilib/misc/config.py index be3bc012..b011ca51 100644 --- a/pyutilib/misc/config.py +++ b/pyutilib/misc/config.py @@ -7,13 +7,13 @@ # the U.S. Government retains certain rights in this software. # _________________________________________________________________________ -""" -================================= +"""================================= The PyUtilib Configuration System ================================= -The PyUtilib config system provides a set of three classes (ConfigDict, -ConfigList, and ConfigValue) for managing and documenting structured +The PyUtilib config system provides a set of three classes +(:py:class:`ConfigDict`, :py:class:`ConfigList`, and +:py:class:`ConfigValue`) for managing and documenting structured configuration information and user input. The system is based around the ConfigValue class, which provides storage for a single configuration entry. ConfigValue objects can be grouped using two containers @@ -75,7 +75,7 @@ >>> print(config.iteration_limit) 20 -All Config objects support a `domain` keyword that accepts a callable +All Config objects support a ``domain`` keyword that accepts a callable object (type, function, or callable instance). The domain callable should take data and map it onto the desired domain, optionally performing domain validation (see :py:class:`ConfigValue`, @@ -95,7 +95,7 @@ ============================= A feature of the Config system is that the core classes all implement -`__call__`, and can themselves be used as `domain` values. Beyond +``__call__``, and can themselves be used as ``domain`` values. Beyond providing domain verification for complex hierarchical structures, this feature allows ConfigDicts to cleanly support the configuration of derived objects. Consider the following example: @@ -125,24 +125,24 @@ filename: input.txt pattern: .*warning -Here, the base class `Base` declares a class-level attribute CONFIG as a -ConfigDict containing a single entry (`filename`). The derived class -(`Derived`) then starts by making a copy of the base class' `CONFIG`, +Here, the base class ``Base`` declares a class-level attribute CONFIG as a +ConfigDict containing a single entry (``filename``). The derived class +(``Derived``) then starts by making a copy of the base class' ``CONFIG``, and then defines an additional entry (`pattern`). Instances of the base -class will still create `c` instances that only have the single -`filename` entry, whereas instances of the derived class will have `c` -instances with two entries: the `pattern` entry declared by the derived -class, and the `filename` entry "inherited" from the base class. +class will still create ``c`` instances that only have the single +``filename`` entry, whereas instances of the derived class will have ``c`` +instances with two entries: the ``pattern`` entry declared by the derived +class, and the ``filename`` entry "inherited" from the base class. An extension of this design pattern provides a clean approach for handling "ephemeral" instance options. Consider an interface to an -external "solver". Our class implements a `solve()` method that takes a +external "solver". Our class implements a ``solve()`` method that takes a problem and sends it to the solver along with some solver configuration options. We would like to be able to set those options "persistently" on instances of the interface class, but still override them -"temporarily" for individual calls to `solve()`. We implement this by +"temporarily" for individual calls to ``solve()``. We implement this by creating copies of the class's configuration for both specific instances -and for use by each `solve()` call: +and for use by each ``solve()`` call: .. doctest:: @@ -177,7 +177,7 @@ class will still create `c` instances that only have the single In addition to basic storage and retrieval, the Config system provides hooks to the argparse command-line argument parsing system. Individual Config entries can be declared as argparse arguments. To make -declaration simpler, the `declare` method returns the declared Config +declaration simpler, the :py:meth:`declare` method returns the declared Config object so that the argument declaration can be done inline: .. doctest:: @@ -261,8 +261,8 @@ class will still create `c` instances that only have the single It is frequently useful to know which values a user explicitly set, and which values a user explicitly set, but have never been retrieved. The configuration system provides two gemerator methods to return the items -that a user explicitly set (`user_values`) and the items that were set -but never retrieved (`unused_user_values`): +that a user explicitly set (:py:meth:`user_values`) and the items that +were set but never retrieved (:py:meth:`unused_user_values`): .. doctest:: @@ -277,11 +277,13 @@ class will still create `c` instances that only have the single ================================= Configuration objects support three methods for generating output and -documentation: `display()`, `generate_yaml_template()`, and -`generate_documentation()`. The simplest is `display()`, which prints -out the current values of the configuration object (and if it is a -container type, all of it's children). `generate_yaml_template` is -simular to `display`, but also includes the description fields as +documentation: :py:meth:`display()`, +:py:meth:`generate_yaml_template()`, and +:py:meth:`generate_documentation()`. The simplest is +:py:meth:`display()`, which prints out the current values of the +configuration object (and if it is a container type, all of it's +children). :py:meth:`generate_yaml_template` is simular to +:py:meth:`display`, but also includes the description fields as formatted comments. .. doctest::