From 06443725cc105631c976cb62342935af012e737f Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 8 Nov 2023 21:47:53 +0100 Subject: [PATCH 01/18] maintenance update --- .editorconfig | 16 ++ .../_build_and_publish_documentation.yml | 2 +- .github/workflows/_build_package.yml | 4 +- .github/workflows/_code_quality.yml | 2 +- .github/workflows/_merge_into_release.yml | 2 +- .gitignore | 16 +- .vscode/extensions.json | 1 + .vscode/launch.json | 32 ++-- .vscode/settings.json | 5 + README.md | 90 +++++++---- STYLEGUIDE.md | 148 +++++++++++------- pyproject.toml | 6 +- requirements-dev.txt | 8 +- requirements-types.txt | 3 +- setup.cfg | 1 + src/dictIO/parser.py | 2 +- tests/conftest.py | 36 ++++- 17 files changed, 247 insertions(+), 127 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..b46c31e5 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = crlf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/_build_and_publish_documentation.yml b/.github/workflows/_build_and_publish_documentation.yml index f01bdb08..e89e8f98 100644 --- a/.github/workflows/_build_and_publish_documentation.yml +++ b/.github/workflows/_build_and_publish_documentation.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout active branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 1 lfs: true diff --git a/.github/workflows/_build_package.yml b/.github/workflows/_build_package.yml index 875b4647..e53ff158 100644 --- a/.github/workflows/_build_package.yml +++ b/.github/workflows/_build_package.yml @@ -7,7 +7,7 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 lfs: true @@ -37,7 +37,7 @@ jobs: # matrix: # platform: [ubuntu-latest, macos-latest, windows-latest] # steps: - # - uses: actions/checkout@v3 + # - uses: actions/checkout@v4 # with: # fetch-depth: 1 # lfs: true diff --git a/.github/workflows/_code_quality.yml b/.github/workflows/_code_quality.yml index 6eaf416b..1f95ef85 100644 --- a/.github/workflows/_code_quality.yml +++ b/.github/workflows/_code_quality.yml @@ -45,6 +45,6 @@ jobs: pip install -r requirements.txt pip install pytest - name: Install pyright - run: pip install pyright>=1.1.325 + run: pip install pyright==1.1.332 - name: Run pyright run: pyright . diff --git a/.github/workflows/_merge_into_release.yml b/.github/workflows/_merge_into_release.yml index 3e45ff8e..cec39643 100644 --- a/.github/workflows/_merge_into_release.yml +++ b/.github/workflows/_merge_into_release.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest environment: release steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Fetch the whole history to prevent unrelated history errors fetch-depth: 0 diff --git a/.gitignore b/.gitignore index 291f07e9..91852394 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,6 @@ __pycache__/ *.py[cod] *$py.class -#vi & emacs -*~ -'#'* - -#packaging -*.whl -*.tar.gz - - -#doxygen -html/ - # C extensions *.so @@ -140,6 +128,9 @@ dmypy.json # Pyre type checker .pyre/ +# PyCharm +.idea + # VS Code Settings .vscode/* !.vscode/settings.json @@ -147,3 +138,4 @@ dmypy.json !.vscode/launch.json !.vscode/extensions.json !.vscode/*.code-snippets + diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 64ba7f8e..b0e4fd78 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -10,6 +10,7 @@ "charliermarsh.ruff", "sourcery.sourcery", "njpwerner.autodocstring", + "editorconfig.editorconfig" ], // List of extensions recommended by VS Code that should not be recommended for users of this workspace. "unwantedRecommendations": [] diff --git a/.vscode/launch.json b/.vscode/launch.json index 3db5af35..5a0a4ee4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,31 +7,35 @@ { "name": "Debug Unit Test", "type": "python", - "request": "test", + "request": "launch", "justMyCode": false, + "program": "${file}" }, { - "name": "Python: Current File", + "name": "Python: Current File, cwd = file dir, envFile", "type": "python", "request": "launch", - "cwd": "${fileDirname}", + "cwd": "${fileDirname}", // working dir = dir where current file is "program": "${file}", "console": "integratedTerminal", - "justMyCode": true, + "justMyCode": false, "autoReload": { "enable": true - } + }, + "envFile": "${workspaceFolder}/.env" // specify where mvx is }, { - "name": "Python: Current File, cwd = workspace root folder", + "name": "Python: Current File, cwd = workspace root folder, envFile", "type": "python", "request": "launch", - "cwd": "${workspaceFolder}", + "cwd": "${workspaceFolder}", // working dir = workspace (mvx) dir "program": "${file}", "console": "integratedTerminal", "autoReload": { "enable": true - } + }, + "justMyCode": false, + "envFile": "${workspaceFolder}/.env" // specify where mvx is }, { "name": "dictParser test_dict", @@ -46,7 +50,9 @@ "console": "integratedTerminal", "autoReload": { "enable": true - } + }, + "justMyCode": false, + "envFile": "${workspaceFolder}/.env" // specify where mvx is }, { "name": "dictParser include\\initialConditions", @@ -61,7 +67,9 @@ "console": "integratedTerminal", "autoReload": { "enable": true - } + }, + "justMyCode": false, + "envFile": "${workspaceFolder}/.env" // specify where mvx is }, { "name": "dictParser test_dict .env", @@ -77,7 +85,9 @@ "console": "integratedTerminal", "autoReload": { "enable": true - } + }, + "justMyCode": false, + "envFile": "${workspaceFolder}/.env" // specify where mvx is }, ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index ac8c9554..6f339ca6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,7 @@ { + "terminal.integrated.env.windows": { + "PYTHONPATH": "${workspaceFolder}/src" + }, "python.terminal.activateEnvInCurrentTerminal": true, "python.languageServer": "Pylance", "ruff.importStrategy": "fromEnvironment", @@ -19,7 +22,9 @@ "python.analysis.diagnosticSeverityOverrides": {}, "python.analysis.indexing": true, "python.analysis.autoImportCompletions": true, + "python.analysis.autoImportUserSymbols": true, "python.analysis.inlayHints.variableTypes": false, "python.analysis.inlayHints.functionReturnTypes": false, "python.analysis.inlayHints.pytestParameters": true, + "python.terminal.executeInFileDir": true, } \ No newline at end of file diff --git a/README.md b/README.md index 087d5384..62d00f4b 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Further, dictIO * can read and write also JSON, XML and OpenFOAM (with some limitations) ## Installation + ```sh pip install dictIO ``` @@ -20,12 +21,12 @@ pip install dictIO ## Usage Example dictIO provides a simple, high level API that allows reading and writing Python dicts from/to dict files: -~~~py +```py from dictIO import DictReader, DictWriter my_dict = DictReader.read('myDict') DictWriter.write(my_dict, 'parsed.myDict') -~~~ +``` The above example reads a dict file, merges any (sub-)dicts included through #include directives, evaluates expressions contained in the dict, and finally saves the read and evaluated dict with prefix 'parsed' as 'parsed.myDict'. @@ -33,16 +34,16 @@ and finally saves the read and evaluated dict with prefix 'parsed' as 'parsed.my This sequence of reading, evaluating and writing a dict is also called 'parsing' in dictIO. Because this task is so common, dictIO provides a convenience class for it: Using DictParser.parse() the above task can be accomplished in one line of code: -~~~py +```py from dictIO import DictParser DictParser.parse('myDict') -~~~ +``` The above task can also be invoked from the command line, using the 'dictParser' command line script installed with dictIO: -~~~sh +```sh dictParser myDict -~~~ +``` _For more examples and usage, please refer to dictIO's [documentation][dictIO_docs]._ @@ -56,43 +57,72 @@ _For a detailed documentation of the dict file format used by dictIO, see [File ## Development Setup -1. Install Python 3.9 or higher, i.e. [Python 3.9](https://www.python.org/downloads/release/python-3912/) or [Python 3.10](https://www.python.org/downloads/release/python-3104/) +1. Install Python 3.9 or higher, i.e. [Python 3.10](https://www.python.org/downloads/release/python-3104/) or [Python 3.11](https://www.python.org/downloads/release/python-3114/) 2. Update pip and setuptools: - ~~~sh - $ python -m pip install --upgrade pip setuptools - ~~~ + ```sh + python -m pip install --upgrade pip setuptools + ``` 3. git clone the dictIO repository into your local development directory: - ~~~sh + ```sh git clone https://github.com/dnv-opensource/dictIO path/to/your/dev/dictIO - ~~~ + ``` 4. In the dictIO root folder: Create a Python virtual environment: - ~~~sh - $ python -m venv .venv - ~~~ - Activate the virtual environment:
+ + ```sh + python -m venv .venv + ``` + + Activate the virtual environment: + ..on Windows: - ~~~sh + + ```sh > .venv\Scripts\activate.bat - ~~~ + ``` + ..on Linux: - ~~~sh - $ source .venv/bin/activate - ~~~ + + ```sh + source .venv/bin/activate + ``` + Update pip and setuptools: - ~~~sh - $ python -m pip install --upgrade pip setuptools - ~~~ + + ```sh + (.venv) $ python -m pip install --upgrade pip setuptools + ``` + Install dictIO's dependencies: - ~~~sh - $ pip install -r requirements-dev.txt - ~~~ + ```sh + (.venv) $ pip install -r requirements-dev.txt + ``` + + This should return without errors. + +5. Setup your development environment to locate Python source codes: + + For example, Visual Studio Code on Windows assumes the Python environment is specified in a `.env` file.
+ If you are developing and running the Python code from VSCode, make sure to create a `.env` file in the mypackage root folder with below content.
+ Set the path for `PROJ_DIR` to where your mypackage folder is on your system.
+ _Note_: `.env` is part of `.gitignore`, such that you do not commit your `.env` file to the repository. + + ```ini + PROJ_DIR= + PYTHONPATH=${PROJ_DIR}/src + ``` + +6. Test that the installation works (in the mypackage root folder): + + ```sh + (.venv) $ pytest . + ``` ## Meta @@ -112,9 +142,9 @@ Distributed under the MIT license. See [LICENSE](LICENSE.md) for more informatio 1. Fork it () 2. Create your branch (`git checkout -b myBranchName`) -3. Commit your changes (`git commit -am 'place your commit message here'`) -4. Push to the branch (`git push origin myBranchName`) -5. Create a new Pull Request +3. Commit your changes (e.g. `git commit -m 'place a descriptive commit message here'`) +4. Push to the branch (e.g. `git push origin myBranchName`) +5. Create a new Pull Request in GitHub For your contribution, please make sure you follow the [STYLEGUIDE](STYLEGUIDE.md) before creating the Pull Request. diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md index 469d81e7..bc79f85f 100644 --- a/STYLEGUIDE.md +++ b/STYLEGUIDE.md @@ -1,8 +1,12 @@ # Style Guide -All code shall be [black](https://pypi.org/project/black/) formatted.
+ +All code shall be [black](https://pypi.org/project/black/) formatted. + References, details as well as examples of bad/good styles and their respective reasoning can be found below. + ## References + * [PEP-8](https://www.python.org/dev/peps/pep-0008/) (see also [pep8.org](https://pep8.org/)) * [PEP-257](https://www.python.org/dev/peps/pep-0257/) * Python style guide by [theluminousmen.com](https://luminousmen.com/post/the-ultimate-python-style-guidelines) @@ -12,54 +16,63 @@ References, details as well as examples of bad/good styles and their respective * [flake8](https://flake8.pycqa.org/en/latest/) * [black](https://pypi.org/project/black/) - ## Code Layout + * Use 4 spaces instead of tabs * Maximum line length is 88 characters (not 79 as proposed in [PEP-8](https://www.python.org/dev/peps/pep-0008/)) * 2 blank lines between classes and functions * 1 blank line within class, between class methods * Use blank lines for logic separation of functionality within functions/methods wherever it is justified * No whitespace adjacent to parentheses, brackets, or braces -~~~py + +```py # Bad spam( items[ 1 ], { key1 : arg1, key2 : arg2 }, ) # Good spam(items[1], {key1: arg1, key2: arg2}, []) -~~~ +``` + * Surround operators with single whitespace on either side. -~~~py + +```py # Bad x<1 # Good x == 1 -~~~ +``` + * Never end your lines with a semicolon, and do not use a semicolon to put two statements on the same line * When branching, always start a new block on a new line -~~~py + +```py # Bad if flag: return None # Good if flag: return None -~~~ +``` + * Similarly to branching, do not write methods on one line in any case: -~~~py + +```py # Bad def do_something(self): print("Something") # Good def do_something(self): print("Something") -~~~ -* Place a class's `__init__` function (the constructor) always at the beginning of the class +``` +* Place a class's `__init__` function (the constructor) always at the beginning of the class ## Line Breaks + * If function arguments do not fit into the specified line length, move them to a new line with indentation -~~~py + +```py # Bad def long_function_name(var_one, var_two, var_three, var_four): @@ -85,9 +98,11 @@ References, details as well as examples of bad/good styles and their respective var_four, ): print(var_one) -~~~ +``` + * Move concatenated logical conditions to new lines if the line does not fit the maximum line size. This will help you understand the condition by looking from top to bottom. Poor formatting makes it difficult to read and understand complex predicates. -~~~py + +```py # Good if ( this_is_one_thing @@ -97,9 +112,11 @@ References, details as well as examples of bad/good styles and their respective and one_more_thing ): do_something() -~~~ +``` + * Where binary operations stretch multiple lines, break lines before the binary operators, not thereafter -~~~py + +```py # Bad GDP = ( private_consumption + @@ -117,9 +134,11 @@ References, details as well as examples of bad/good styles and their respective + government_spending + (exports - imports) ) -~~~ +``` + * Chaining methods should be broken up on multiple lines for better readability -~~~py + +```py ( df.write.format("jdbc") .option("url", "jdbc:postgresql:dbserver") @@ -128,9 +147,11 @@ References, details as well as examples of bad/good styles and their respective .option("password", "password") .save() ) -~~~ +``` + * Add a trailing comma to sequences of items when the closing container token ], ), or } does not appear on the same line as the final element -~~~py + +```py # Bad y = [ 0, @@ -157,29 +178,32 @@ References, details as well as examples of bad/good styles and their respective 'a': 1, 'b': 2, <- note the trailing comma } -~~~ - +``` ## String Formatting + * When quoting string literals, use double-quoted strings. When the string itself contains single or double quote characters, however, use the respective other one to avoid backslashes in the string. It improves readability. * Use f-strings to format strings: -~~~py + +```py # Bad print("Hello, %s. You are %s years old. You are a %s." % (name, age, profession)) # Good print(f"Hello, {name}. You are {age} years old. You are a {profession}.") -~~~ +``` + * Use multiline strings, not \ , since it gets much more readable. -~~~py + +```py raise AttributeError( "Here is a multiline error message with a very long first line " "and a shorter second line." ) -~~~ - +``` ## Naming Conventions + * For module names: `lowercase` . Long module names can have words separated by underscores (`really_long_module_name.py`), but this is not required. Try to use the convention of nearby files. * For class names: `CamelCase` @@ -192,19 +216,22 @@ Long module names can have words separated by underscores (`really_long_module_n * Names shall be clear about what a variable, class, or function contains or does. If you struggle to come up with a clear name, rethink your architecture: Often, the difficulty in finding a crisp name for something is a hint that separation of responsibilities can be improved. The solution then is less to agree on a name, but to start a round of refactoring: The name you're seeking often comes naturally then with refactoring to an improved architecture with clear responsibilities. (see [SRP](https://en.wikipedia.org/wiki/Single-responsibility_principle), Single-Responsibilty Principle by Robert C. Martin) - ## Named Arguments + * Use named arguments to improve readability and avoid mistakes introduced with future code maintenance -~~~py + +```py # Bad urlget("[http://google.com](http://google.com/)", 20) # Good urlget("[http://google.com](http://google.com/)", timeout=20) -~~~ +``` + * Never use mutable objects as default arguments in Python. If an attribute in a class or a named parameter in a function is of a mutable data type (e.g. a list or dict), never set its default value in the declaration of an object but always set it to None first, and then only later assign the default value in the class's constructor, or the functions body, respectively. Sounds complicated? If you prefer the shortcut, the examples below are your friend. If your are interested in the long story including the why‘s, read these discussions on [Reddit](https://old.reddit.com/r/Python/comments/opb7hm/do_not_use_mutable_objects_as_default_arguments/) and [Twitter](https://twitter.com/willmcgugan/status/1419616480971399171). -~~~py + +```py # Bad class Foo: items = [] @@ -235,25 +262,30 @@ If your are interested in the long story including the why‘s, read these discu def some_function(x, y, items=None): items = items or [] ... -~~~ - +``` ## Commenting + * First of all, if the code needs comments to clarify its work, you should think about refactoring it. The best comment to code is the code itself. * Describe complex, possibly incomprehensible points and side effects in the comments * Separate `#` and the comment with one whitespace -~~~py + +```py #bad comment # good comment -~~~ +``` + * Use inline comments sparsely * Where used, inline comments shall have 2 whitespaces before the `#` and one whitespace thereafter -~~~py + +```py x = y + z # inline comment str1 = str2 + str3 # another inline comment -~~~ +``` + * If a piece of code is poorly understood, mark the piece with a `@TODO:` tag and your name to support future refactoring: -~~~py + +```py def get_ancestors_ids(self): # @TODO: Do a cache reset while saving the category tree. CLAROS, YYYY-MM-DD cache_name = f"{self._meta.model_name}_ancestors_{self.pk}" @@ -265,20 +297,21 @@ If your are interested in the long story including the why‘s, read these discu cache.set(cache_name, ids, timeout=3600) return ids -~~~ - +``` ## Type hints + * Use type hints in function signatures and module-scope variables. This is good documentation and can be used with linters for type checking and error checking. Use them whenever possible. * Use pyi files to type annotate third-party or extension modules. - ## Docstrings + * All Docstrings should be written in [Numpy](https://numpydoc.readthedocs.io/en/latest/format.html) format. For a good tutorial on Docstrings, see [Documenting Python Code: A Complete Guide](https://realpython.com/documenting-python-code) * In a Docstring, summarize function/method behavior and document its arguments, return value(s), side effects, exceptions raised, and restrictions * Wrap Docstrings with triple double quotes (""") * The description of the arguments must be indented -~~~py + +```py def some_method(name, print=False): """This function does something @@ -301,21 +334,23 @@ If your are interested in the long story including the why‘s, read these discu """ ... return 0 -~~~ +``` ## Exceptions + * Raise specific exceptions and catch specific exceptions, such as KeyError, ValueError, etc. * Do not raise or catch just Exception, except in rare cases where this is unavoidable, such as a try/except block on the top-level loop of some long-running process. For a good tutorial on why this matters, see [The Most Diabolical Python Antipattern](https://realpython.com/the-most-diabolical-python-antipattern/). * Minimize the amount of code in a try/except block. The larger the body of the try, the more likely that an exception will be raised by a line of code that you didn’t expect to raise an exception. - ## Imports + * Avoid creating circular imports by importing modules more specialized than the one you are editing * Relative imports are forbidden ([PEP-8](https://www.python.org/dev/peps/pep-0008/) only “highly discourages” them). Where absolutely needed, the `from future import absolute_import` syntax should be used (see [PEP-328](https://www.python.org/dev/peps/pep-0328/)) * Never use wildcard imports (`from import *`). Always be explicit about what you're importing. Namespaces make code easier to read, so use them. * Break long imports using parentheses and indent by 4 spaces. Include the trailing comma after the last import and place the closing bracket on a separate line -~~~py + +```py from my_pkg.utils import ( some_utility_method_1, some_utility_method_2, @@ -323,12 +358,14 @@ If your are interested in the long story including the why‘s, read these discu some_utility_method_4, some_utility_method_5, ) -~~~ +``` + * Imports should be written in the following order, separated by a blank line: 1. build-in modules 2. third-party modules 3. local application/library specific imports -~~~py + +```py import logging import os import typing as T @@ -339,22 +376,25 @@ If your are interested in the long story including the why‘s, read these discu import my_package import my_package.my_module from my_package.my_module import my_function, MyClass -~~~ +``` + * Even if a Python file is intended to be used as executable / script file only, it shall still be importable as a module, and its import should not have any side effects. Its main functionality shall hence be in a `main()` function, so that the code can be imported as a module for testing or being reused in the future: -~~~py + +```py def main(): ... if __name__ == "__main__": main() -~~~ - +``` ## Unit-tests + * Use pytest as the preferred testing framework. * The name of a test shall clearly express what is being tested. * Each test should preferably check only one specific aspect. -~~~py + +```py # Bad def test_smth(): result = f() @@ -372,10 +412,10 @@ If your are interested in the long story including the why‘s, read these discu def test_smth_values(): result = f() assert set(result) == set(expected), f"Result should be {set(expected)}" -~~~ +``` +## And finally: It is a bad idea to use -## And finally: It is a bad idea to use.. * global variables. * iterators where they can be replaced by vectorized operations. * lambda where it is not required. diff --git a/pyproject.toml b/pyproject.toml index 1f96ce0b..1533511d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["lxml", "setuptools", "wheel"] +requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [tool.black] @@ -70,6 +70,7 @@ exclude = [ "dist", "**/__pycache__", "./docs/source/conf.py", + "./venv", ] extraPaths = ["./src"] typeCheckingMode = "basic" @@ -109,4 +110,5 @@ reportUntypedNamedTuple = "warning" [tool.pytest.ini_options] testpaths = "tests" addopts = "--strict-markers" -xfail_strict = true \ No newline at end of file +xfail_strict = true +pythonpath = ["src"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 5521e044..50e4d776 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,14 +1,14 @@ pytest>=7.4 pytest-cov>=4.1 pytest-randomly>=3.15 -black[jupyter]>=23.9 -ruff>=0.0.290 -pyright==1.1.325 +black[jupyter]>=23.10 +ruff>=0.1.2 +pyright==1.1.332 Sphinx>=7.2 sphinx-argparse-cli>=1.11 myst-parser>=2.0 furo>=2023.9.10 -sourcery>=1.9.0 +sourcery>=1.12.0 -r requirements.txt -r requirements-types.txt diff --git a/requirements-types.txt b/requirements-types.txt index 73b93104..e4e98ed5 100644 --- a/requirements-types.txt +++ b/requirements-types.txt @@ -1,2 +1 @@ -types-lxml>=2023.3.28 -# lxml-stubs>=0.4.0 +types-lxml>=2023.10.21 diff --git a/setup.cfg b/setup.cfg index 26f849cf..988d8456 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,6 +77,7 @@ envlist = py{39,310,311}-{linux,macos,windows} # envlist = py{39,310,311} [testenv] +system_site_packages = True deps = pytest>=7.4 pytest-cov>=4.1 diff --git a/src/dictIO/parser.py b/src/dictIO/parser.py index 2d76a980..2cb681df 100644 --- a/src/dictIO/parser.py +++ b/src/dictIO/parser.py @@ -638,7 +638,7 @@ def _extract_expressions(self, dict: CppDict): # Register the expression in .expressions expression = re.sub(r"\"", "", expression) - dict.expressions.update({index: {"expression": expression, "name": placeholder}}) + dict.expressions |= {index: {"expression": expression, "name": placeholder}} # Step 2: Find references in .block_content (single references to key'd entries that are NOT in double quotes). search_pattern = r"\$\w[\w\[\]]+" diff --git a/tests/conftest.py b/tests/conftest.py index 4d29602c..ca9bb5b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,11 @@ +import logging import os from glob import glob from pathlib import Path +from shutil import rmtree import pytest +from pytest import LogCaptureFixture @pytest.fixture(scope="package", autouse=True) @@ -10,18 +13,39 @@ def chdir(): os.chdir(Path(__file__).parent.absolute() / "test_dicts") -dictIO_files = ["parsed.*"] # noqa +@pytest.fixture(scope="package", autouse=True) +def test_dir(): + return Path(__file__).parent.absolute() + + +output_dirs = [] +output_files = [ + "parsed.*", +] @pytest.fixture(autouse=True) -def default_setup_and_teardown(): - _remove_dictIO_files() +def default_setup_and_teardown(caplog: LogCaptureFixture): + _remove_output_dirs_and_files() yield - _remove_dictIO_files() + _remove_output_dirs_and_files() -def _remove_dictIO_files(): # noqa - for pattern in dictIO_files: +def _remove_output_dirs_and_files(): + for folder in output_dirs: + rmtree(folder, ignore_errors=True) + for pattern in output_files: for file in glob(pattern): file = Path(file) file.unlink(missing_ok=True) + + +@pytest.fixture(autouse=True) +def setup_logging(caplog: LogCaptureFixture): + caplog.set_level("INFO") + caplog.clear() + + +@pytest.fixture(autouse=True) +def logger(): + return logging.getLogger() From 8214311ddf7b4021f1f6037c5ea0c0130ea76d31 Mon Sep 17 00:00:00 2001 From: Claas Date: Thu, 9 Nov 2023 17:04:51 +0100 Subject: [PATCH 02/18] maintenance update --- .gitignore | 3 +++ .vscode/launch.json | 10 +++++----- README.md | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 91852394..ba7ab27e 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,9 @@ dmypy.json # PyCharm .idea +# modules +modules.txt + # VS Code Settings .vscode/* !.vscode/settings.json diff --git a/.vscode/launch.json b/.vscode/launch.json index 5a0a4ee4..df4cda5f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,7 +22,7 @@ "autoReload": { "enable": true }, - "envFile": "${workspaceFolder}/.env" // specify where mvx is + "envFile": "${workspaceFolder}/.env" // specify where .env file is }, { "name": "Python: Current File, cwd = workspace root folder, envFile", @@ -35,7 +35,7 @@ "enable": true }, "justMyCode": false, - "envFile": "${workspaceFolder}/.env" // specify where mvx is + "envFile": "${workspaceFolder}/.env" // specify where .env file is }, { "name": "dictParser test_dict", @@ -52,7 +52,7 @@ "enable": true }, "justMyCode": false, - "envFile": "${workspaceFolder}/.env" // specify where mvx is + "envFile": "${workspaceFolder}/.env" // specify where .env file is }, { "name": "dictParser include\\initialConditions", @@ -69,7 +69,7 @@ "enable": true }, "justMyCode": false, - "envFile": "${workspaceFolder}/.env" // specify where mvx is + "envFile": "${workspaceFolder}/.env" // specify where .env file is }, { "name": "dictParser test_dict .env", @@ -87,7 +87,7 @@ "enable": true }, "justMyCode": false, - "envFile": "${workspaceFolder}/.env" // specify where mvx is + "envFile": "${workspaceFolder}/.env" // specify where .env file is }, ] } \ No newline at end of file diff --git a/README.md b/README.md index 62d00f4b..277a9264 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ _For a detailed documentation of the dict file format used by dictIO, see [File _Note_: `.env` is part of `.gitignore`, such that you do not commit your `.env` file to the repository. ```ini - PROJ_DIR= + PROJ_DIR= PYTHONPATH=${PROJ_DIR}/src ``` From 969aa77f4c8ff93f84d627f8ed30555cc7cfa24e Mon Sep 17 00:00:00 2001 From: Claas Date: Fri, 17 Nov 2023 09:53:13 +0100 Subject: [PATCH 03/18] maintenance update --- .vscode/extensions.json | 2 +- .vscode/launch.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index b0e4fd78..a4d80c37 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -10,7 +10,7 @@ "charliermarsh.ruff", "sourcery.sourcery", "njpwerner.autodocstring", - "editorconfig.editorconfig" + "editorconfig.editorconfig", ], // List of extensions recommended by VS Code that should not be recommended for users of this workspace. "unwantedRecommendations": [] diff --git a/.vscode/launch.json b/.vscode/launch.json index df4cda5f..6b91b5b0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,10 +18,10 @@ "cwd": "${fileDirname}", // working dir = dir where current file is "program": "${file}", "console": "integratedTerminal", - "justMyCode": false, "autoReload": { "enable": true }, + "justMyCode": false, "envFile": "${workspaceFolder}/.env" // specify where .env file is }, { @@ -75,7 +75,6 @@ "name": "dictParser test_dict .env", "type": "python", "request": "launch", - "envFile": "${workspaceFolder}\\.env", "cwd": "${workspaceFolder}\\tests", "program": "${workspaceFolder}\\src\\dictIO\\cli\\dictParser.py", "args": [ From 172b7217b01843c2a5899d2566e542a0c82a847d Mon Sep 17 00:00:00 2001 From: Claas Date: Sun, 19 Nov 2023 18:53:36 +0100 Subject: [PATCH 04/18] updated dependencies --- .github/workflows/_build_package.yml | 2 +- .github/workflows/_code_quality.yml | 6 +++--- requirements-dev.txt | 8 ++++---- requirements.txt | 4 ++-- setup.cfg | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_build_package.yml b/.github/workflows/_build_package.yml index e53ff158..7872bc74 100644 --- a/.github/workflows/_build_package.yml +++ b/.github/workflows/_build_package.yml @@ -46,7 +46,7 @@ jobs: # python-version: '3.11' # cache: 'pip' # cache pip dependencies # - name: Install cibuildwheel - # run: python -m pip install cibuildwheel==2.3.1 + # run: python -m pip install cibuildwheel==2.16 # - name: Build wheels # run: python -m cibuildwheel --output-dir wheels # - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/_code_quality.yml b/.github/workflows/_code_quality.yml index 1f95ef85..f155ab83 100644 --- a/.github/workflows/_code_quality.yml +++ b/.github/workflows/_code_quality.yml @@ -13,7 +13,7 @@ jobs: options: '--check --diff' src: '.' jupyter: true - version: '>=23.9' + version: '==23.11' ruff: runs-on: ubuntu-latest @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: pip install -r requirements.txt - name: Install ruff - run: pip install ruff>=0.0.290 + run: pip install ruff==0.1.6 - name: Run ruff run: ruff . @@ -45,6 +45,6 @@ jobs: pip install -r requirements.txt pip install pytest - name: Install pyright - run: pip install pyright==1.1.332 + run: pip install pyright==1.1.336 - name: Run pyright run: pyright . diff --git a/requirements-dev.txt b/requirements-dev.txt index 50e4d776..d9188985 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,14 +1,14 @@ pytest>=7.4 pytest-cov>=4.1 pytest-randomly>=3.15 -black[jupyter]>=23.10 -ruff>=0.1.2 -pyright==1.1.332 +black[jupyter]==23.11 +ruff==0.1.6 +pyright==1.1.336 Sphinx>=7.2 sphinx-argparse-cli>=1.11 myst-parser>=2.0 furo>=2023.9.10 -sourcery>=1.12.0 +sourcery==1.14 -r requirements.txt -r requirements-types.txt diff --git a/requirements.txt b/requirements.txt index 27ee99c7..6f178443 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ lxml>=4.9 -jsonschema>=4.19 -numpy>=1.24 +jsonschema>=4.20 +numpy>=1.26 diff --git a/setup.cfg b/setup.cfg index 988d8456..90831399 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,8 +39,8 @@ include_package_data = True python_requires = >=3.9 install_requires = lxml>=4.9 - jsonschema>=4.19 - numpy>=1.24 + jsonschema>=4.20 + numpy>=1.26 [options.packages.find] where = src From 5078eaa25a8f9b73e848c9d957e251c2df468a97 Mon Sep 17 00:00:00 2001 From: Claas Date: Mon, 20 Nov 2023 08:55:55 +0100 Subject: [PATCH 05/18] resolved issues raised by pyright 1.1.336 --- src/dictIO/formatter.py | 2 +- tests/test_dictReader.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dictIO/formatter.py b/src/dictIO/formatter.py index 5a682896..20f12441 100644 --- a/src/dictIO/formatter.py +++ b/src/dictIO/formatter.py @@ -396,7 +396,7 @@ def to_string( dict.data = sorted_data # Create the string representation of the dictionary in its basic structure. - s += self.format_dict(dict.data) + s += self.format_dict(dict.data) # type: ignore # The following elements a CppDict's .data attribute # are usually still substituted by placeholders: diff --git a/tests/test_dictReader.py b/tests/test_dictReader.py index 4615db37..dd5cf19b 100644 --- a/tests/test_dictReader.py +++ b/tests/test_dictReader.py @@ -248,21 +248,21 @@ def test_eval_expressions_with_included_numpy_expressions(): dict_out = dict.data assert dict_out["keysContainingNumpyExpressions"]["npKeyA"] == 2 - assert_array_equal(dict_out["keysContainingNumpyExpressions"]["npKeyB"], [[1, 1], [1, 1]]) + assert_array_equal(dict_out["keysContainingNumpyExpressions"]["npKeyB"], [[1, 1], [1, 1]]) # type: ignore assert_array_equal(dict_out["keysContainingNumpyExpressions"]["npKeyC"], [2, 2, 2]) assert_array_equal( dict_out["keysContainingNumpyExpressions"]["npKeyD"], - [[2, 0, 0], [0, 2, 0], [0, 0, 2]], + [[2, 0, 0], [0, 2, 0], [0, 0, 2]], # type: ignore ) assert_array_equal( dict_out["keysContainingNumpyExpressions"]["npKeyE"], - [[2, 0, 0], [0, 2, 0], [0, 0, 2]], + [[2, 0, 0], [0, 2, 0], [0, 0, 2]], # type: ignore ) - assert_array_equal(dict_out["keysContainingNumpyExpressions"]["npKeyZ"], [[0, 0, 0, 0]]) + assert_array_equal(dict_out["keysContainingNumpyExpressions"]["npKeyZ"], [[0, 0, 0, 0]]) # type: ignore def test_reread_string_literals(): From 31515bb71bb90c045a24d9681a99041d73205361 Mon Sep 17 00:00:00 2001 From: Claas Date: Tue, 19 Dec 2023 22:23:27 +0100 Subject: [PATCH 06/18] updated VS Code settings --- .vscode/settings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6f339ca6..52762a52 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,7 +11,7 @@ "[python]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, }, "autoDocstring.docstringFormat": "numpy", @@ -27,4 +27,4 @@ "python.analysis.inlayHints.functionReturnTypes": false, "python.analysis.inlayHints.pytestParameters": true, "python.terminal.executeInFileDir": true, -} \ No newline at end of file +} From 44257e68141338db76fbaeb2210438b0d7404db4 Mon Sep 17 00:00:00 2001 From: Claas Date: Tue, 19 Dec 2023 22:24:09 +0100 Subject: [PATCH 07/18] updated versions of black, ruff, pyright updated to black[jupyter]==23.12 (from black[jupyter]==23.11) updated to ruff==0.1.8 (from ruff==0.1.6) updated to pyright==1.1.338 (from pyright==1.1.336) --- .github/workflows/_code_quality.yml | 6 +++--- requirements-dev.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_code_quality.yml b/.github/workflows/_code_quality.yml index f155ab83..2ee82477 100644 --- a/.github/workflows/_code_quality.yml +++ b/.github/workflows/_code_quality.yml @@ -13,7 +13,7 @@ jobs: options: '--check --diff' src: '.' jupyter: true - version: '==23.11' + version: '==23.12' ruff: runs-on: ubuntu-latest @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: pip install -r requirements.txt - name: Install ruff - run: pip install ruff==0.1.6 + run: pip install ruff==0.1.8 - name: Run ruff run: ruff . @@ -45,6 +45,6 @@ jobs: pip install -r requirements.txt pip install pytest - name: Install pyright - run: pip install pyright==1.1.336 + run: pip install pyright==1.1.338 - name: Run pyright run: pyright . diff --git a/requirements-dev.txt b/requirements-dev.txt index d9188985..facdf688 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,9 @@ pytest>=7.4 pytest-cov>=4.1 pytest-randomly>=3.15 -black[jupyter]==23.11 -ruff==0.1.6 -pyright==1.1.336 +black[jupyter]==23.12 +ruff==0.1.8 +pyright==1.1.338 Sphinx>=7.2 sphinx-argparse-cli>=1.11 myst-parser>=2.0 From 61801fba048b9bcaa90b67b239e0c6cba75f65a4 Mon Sep 17 00:00:00 2001 From: Claas Date: Tue, 19 Dec 2023 22:32:54 +0100 Subject: [PATCH 08/18] GitHub workflows: Include Python 3.12 release version as standard, and Python 3.13.0a2 as "future" test. --- .github/workflows/_test.yml | 2 ++ .github/workflows/_test_future.yml | 4 ++-- pyproject.toml | 2 +- setup.cfg | 7 ++++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index b78e2ad8..8b21b411 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -20,6 +20,8 @@ jobs: toxenv: 'py310' - version: '3.11' toxenv: 'py311' + - version: '3.12' + toxenv: 'py312' steps: - uses: actions/checkout@v4 - name: Install Python ${{ matrix.python.version }} diff --git a/.github/workflows/_test_future.yml b/.github/workflows/_test_future.yml index 49405a51..5d07fee3 100644 --- a/.github/workflows/_test_future.yml +++ b/.github/workflows/_test_future.yml @@ -16,8 +16,8 @@ jobs: - runner: windows-latest toxenv: windows python: - - version: '3.12.0rc3' - toxenv: 'py312' + - version: '3.13.0a2' + toxenv: 'py313' steps: - uses: actions/checkout@v4 - name: Install Python ${{ matrix.python.version }} diff --git a/pyproject.toml b/pyproject.toml index 1533511d..c15f6402 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 120 -target-version = ["py39", "py310"] +target-version = ["py39", "py310", "py311", "py312"] [tool.ruff] exclude = [ diff --git a/setup.cfg b/setup.cfg index 90831399..77929844 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Operating System :: Microsoft :: Windows Operating System :: POSIX :: Linux Operating System :: MacOS @@ -72,9 +73,9 @@ source = [tox:tox] isolated_build = True -envlist = py{39,310,311}-{linux,macos,windows} -# envlist = py{39,310,311}-{windows} -# envlist = py{39,310,311} +envlist = py{39,310,311,312}-{linux,macos,windows} +# envlist = py{39,310,311,312}-{windows} +# envlist = py{39,310,311,312} [testenv] system_site_packages = True From e0476b5b8d0c2d39ed3261967982ff5aaf75ace8 Mon Sep 17 00:00:00 2001 From: Claas Date: Tue, 19 Dec 2023 22:55:51 +0100 Subject: [PATCH 09/18] deleted orphaned modules.txt --- modules.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 modules.txt diff --git a/modules.txt b/modules.txt deleted file mode 100644 index 0127a78f..00000000 --- a/modules.txt +++ /dev/null @@ -1,5 +0,0 @@ -_cmd -_dev -python/python39 -python/venvs/pythonBuild - From a06b117c90b1e9d11dd7b4ec2516b4f259ebea50 Mon Sep 17 00:00:00 2001 From: Claas Date: Thu, 21 Dec 2023 17:16:36 +0100 Subject: [PATCH 10/18] parser and formatter: added handling of strings with nested strings --- src/dictIO/formatter.py | 60 +++++++++++++++++++++ src/dictIO/parser.py | 110 ++++++++++++++++++++++++++++++--------- tests/test_dictReader.py | 39 ++++++++++++++ tests/test_formatter.py | 4 ++ 4 files changed, 187 insertions(+), 26 deletions(-) diff --git a/src/dictIO/formatter.py b/src/dictIO/formatter.py index 20f12441..c2002c7b 100644 --- a/src/dictIO/formatter.py +++ b/src/dictIO/formatter.py @@ -217,6 +217,8 @@ def format_string(self, arg: str) -> str: return self.format_expression_string(arg) elif not arg: # empty string return self.format_empty_string(arg) + elif re.search(r"[\"']", arg): # contains a nested string + return self.format_string_with_nested_string(arg) elif re.search(r"[\s:/\\]", arg): # contains spaces or path -> complex string return self.format_multi_word_string(arg) else: # single word string @@ -256,6 +258,23 @@ def format_single_word_string(self, arg: str) -> str: """ return arg + def format_string_with_nested_string(self, arg: str) -> str: + """Format a string that contains a nested string. + + Note: Override this method for specific formatting of strings with nested strings when implementing a Formatter. + + Parameters + ---------- + arg : str + the string with a nested string to be formatted + + Returns + ------- + str + the formatted string with a nested string + """ + return self.add_single_quotes(arg) + def format_multi_word_string(self, arg: str) -> str: """Format a multi word string. @@ -607,6 +626,26 @@ def format_empty_string(self, arg: str) -> str: """ return self.add_single_quotes(arg) + def format_string_with_nested_string(self, arg: str) -> str: + """Format a string that contains a nested string. + + Parameters + ---------- + arg : str + the string with a nested string to be formatted + + Returns + ------- + str + the formatted string with a nested string + """ + if re.search(r'"', arg): + return self.add_single_quotes(arg) + elif re.search(r"'", arg): + return self.add_double_quotes(arg) + else: + raise ValueError(f"expected a string with a nested string. However, following string was passed in: {arg}") + def format_multi_word_string(self, arg: str) -> str: """Format a multi word string. @@ -801,6 +840,27 @@ def format_empty_string(self, arg: str) -> str: """ return self.add_double_quotes(arg) + def format_string_with_nested_string(self, arg: str) -> str: + """Format a string that contains a nested string. + + Parameters + ---------- + arg : str + the string with a nested string to be formatted + + Returns + ------- + str + the formatted string with a nested string + """ + if re.search(r'"', arg): + _arg: str = re.sub(r'"', '\\"', arg) + return self.add_double_quotes(_arg) + elif re.search(r"'", arg): + return self.add_double_quotes(arg) + else: + raise ValueError(f"expected a string with a nested string. However, following string was passed in: {arg}") + def format_multi_word_string(self, arg: str) -> str: """Format a multi word string. diff --git a/src/dictIO/parser.py b/src/dictIO/parser.py index 2cb681df..2e74b42c 100644 --- a/src/dictIO/parser.py +++ b/src/dictIO/parser.py @@ -2,6 +2,7 @@ import os import re from pathlib import Path +from re import Match from typing import ( Any, Dict, @@ -313,7 +314,7 @@ def remove_quotes_from_string(arg: str, all: bool = False) -> str: @staticmethod def remove_quotes_from_strings( - arg: Union[MutableMapping[Any, Any], MutableSequence[Any]] + arg: Union[MutableMapping[Any, Any], MutableSequence[Any]], ) -> Union[MutableMapping[Any, Any], MutableSequence[Any]]: """Remove quotes from multiple strings. @@ -563,8 +564,87 @@ def _extract_string_literals(self, dict: CppDict): string_literals: List[str] # Step 1: Find single quoted string literals in .block_content - search_pattern = r"\'.*?\'" - string_literals = re.findall(search_pattern, dict.block_content, re.MULTILINE) + search_pattern = re.compile( + pattern=r"(?P((?((? sq_start and dq_start < sq_end: + # sq match is inside dq match -> sq match is nested + found_nested_in_double_quoted_match = True + break + if found_nested_in_double_quoted_match: + _single_quoted_string_literals_found_nested.append(single_quoted_string_literal) + else: + _single_quoted_string_literals_not_nested.append(single_quoted_string_literal) + + # Classify all double quoted string literals as to whether they are (also) + # found as a nested literal in any single quoted string literal, or not. + _double_quoted_string_literals_found_nested: List[str] = [] + _double_quoted_string_literals_not_nested: List[str] = [] + for double_quoted_match in double_quoted_matches: + double_quoted_string_literal: str = double_quoted_match.string[ + double_quoted_match.start(0) : double_quoted_match.end(0) + ] + dq_start: int = double_quoted_match.start(0) + sq_start: int + sq_end: int + found_nested_in_single_quoted_match: bool = False + for single_quoted_match in single_quoted_matches: + sq_start = single_quoted_match.start(0) + sq_end = single_quoted_match.end(0) + if dq_start > sq_start and dq_start < sq_end: + # dq match is inside sq match -> dq match is nested + found_nested_in_single_quoted_match = True + break + if found_nested_in_single_quoted_match: + _double_quoted_string_literals_found_nested.append(double_quoted_string_literal) + else: + _double_quoted_string_literals_not_nested.append(double_quoted_string_literal) + + # For replacement of the string literals inside dict.block_content: + # Chain the different identified string literals in such a sequence that + # outer literals (i.e. those that are NOT found nested inside another literal) + # are replaced first. String literals that were found (also) nested inside other literals + # are replaced last. The latter then would only replace occurences of these string literals + # where they are NOT nested (as the nested occurences are already replaced by + # the placeholders of the outer string literals they were nested in). + string_literals: List[str] = ( + _single_quoted_string_literals_not_nested + + _double_quoted_string_literals_not_nested + + _single_quoted_string_literals_found_nested + + _double_quoted_string_literals_found_nested + ) for string_literal in string_literals: index = self.counter() placeholder = "STRINGLITERAL%06i" % index @@ -572,33 +652,11 @@ def _extract_string_literals(self, dict: CppDict): # Replace all occurances of the string literal in .block_content with the placeholder (STRINGLITERAL000000) # Note: For re.sub() to work properly we need to escape all special characters search_pattern = re.compile(re.escape(string_literal)) - dict.block_content = re.sub( - search_pattern, placeholder, dict.block_content - ) # replace string literal in .block_content with placeholder + dict.block_content = re.sub(search_pattern, placeholder, dict.block_content) # Register the string literal in .string_literals dict.string_literals.update({index: __class__.remove_quotes_from_string(string_literal)}) - # Step 2: Find double quoted string literals in .block_content - # Double quoted strings are identified as string literals only in case they do not contain a $ character. - # (double quoted strings containing a $ character are considered expressions, not string literals.) - search_pattern = r"\".*?\"" - string_literals = re.findall(search_pattern, dict.block_content, re.MULTILINE) - for string_literal in string_literals: - if "$" not in string_literal: - index = self.counter() - placeholder = "STRINGLITERAL%06i" % index - - # Replace all occurances of the string literal in .block_content with the placeholder (STRINGLITERAL000000) - # Note: For re.sub() to work properly we need to escape all special characters - search_pattern = re.compile(re.escape(string_literal)) - dict.block_content = re.sub( - search_pattern, placeholder, dict.block_content - ) # replace expression in .block_content with placeholder - - # Register the string literal in .string_literals - dict.string_literals.update({index: __class__.remove_quotes_from_string(string_literal)}) - return def _extract_expressions(self, dict: CppDict): diff --git a/tests/test_dictReader.py b/tests/test_dictReader.py index dd5cf19b..08d57074 100644 --- a/tests/test_dictReader.py +++ b/tests/test_dictReader.py @@ -523,6 +523,45 @@ def test_read_circular_includes_log_warning(caplog: LogCaptureFixture): assert caplog.records[0].message == log_message_expected +def test_read_strings_dict(): + # sourcery skip: avoid-builtin-shadow + # Prepare + source_file = Path("test_strings_dict") + # Execute + dict = DictReader.read(source_file) + # Assert strings are parsed correctly + assert dict["subDict"]["string_00_dq_empty"] == "" + assert dict["subDict"]["string_01_sq_empty"] == "" + assert dict["subDict"]["string_02_dq_word"] == "string_02_dq_word" + assert dict["subDict"]["string_03_sq_word"] == "string_03_sq_word" + assert dict["subDict"]["string_04_dq_sq_word"] == r"quote('string_04_dq_sq_word')" + assert dict["subDict"]["string_05_dq_escsq_word"] == r"quote(\'string_05_dq_escsq_word\')" + assert dict["subDict"]["string_06_sq_dq_word"] == r'quote("string_06_sq_dq_word")' + assert dict["subDict"]["string_07_sq_escdq_word"] == r"quote(\"string_07_sq_escdq_word\")" + + +def test_reread_strings_dict(): + # sourcery skip: avoid-builtin-shadow + # Prepare + source_file = Path("test_strings_dict") + parsed_file = Path(f"parsed.{source_file.name}") + parsed_file.unlink(missing_ok=True) + # Execute + dict = DictReader.read(source_file) + DictWriter.write(dict, parsed_file) + assert parsed_file.exists() + # Assert strings are parsed correctly + reread_dict = DictReader.read(parsed_file) + assert reread_dict["subDict"]["string_00_dq_empty"] == "" + assert reread_dict["subDict"]["string_01_sq_empty"] == "" + assert reread_dict["subDict"]["string_02_dq_word"] == "string_02_dq_word" + assert reread_dict["subDict"]["string_03_sq_word"] == "string_03_sq_word" + assert reread_dict["subDict"]["string_04_dq_sq_word"] == r"quote('string_04_dq_sq_word')" + assert reread_dict["subDict"]["string_05_dq_escsq_word"] == r"quote(\'string_05_dq_escsq_word\')" + assert reread_dict["subDict"]["string_06_sq_dq_word"] == r'quote("string_06_sq_dq_word")' + assert reread_dict["subDict"]["string_07_sq_escdq_word"] == r"quote(\"string_07_sq_escdq_word\")" + + class SetupHelper: @staticmethod def prepare_dict_until( diff --git a/tests/test_formatter.py b/tests/test_formatter.py index b49c5cf6..af12141d 100644 --- a/tests/test_formatter.py +++ b/tests/test_formatter.py @@ -48,6 +48,8 @@ def test_format_type_string_no_additional_quotes_expected(self, str_in: str): r"C:\a\path\in\windows", r"C:/a/path/in/linux", "", + r'contains a "nested string" literal with double quotes', + r"contains a \"nested string\" literal with escaped double quotes", ], ) def test_format_type_string_additional_single_quotes_expected(self, str_in: str): @@ -65,6 +67,8 @@ def test_format_type_string_additional_single_quotes_expected(self, str_in: str) "$keyword+1", "$keyword - 3.0", "$keyword1 * $keyword2", + r"contains a 'nested string' literal with single quotes", + r"contains a \'nested string\' literal with escaped single quotes", ], ) def test_format_type_string_additional_double_quotes_expected(self, str_in: str): From 77be17f4c2d94b4d6240538059a5d488f33a5ce9 Mon Sep 17 00:00:00 2001 From: Claas Date: Fri, 22 Dec 2023 09:55:48 +0100 Subject: [PATCH 11/18] updated CHANGELOG.md --- CHANGELOG.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4126866..1b9ba030 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/e * -/- + +## [0.3.0] - 2023-12-22 + +### Changed + +* Enabled recognition of strings with nested quotes in it (solves [#2](https://github.com/dnv-opensource/dictIO/issues/2)) +* GitHub workflows: Included Python 3.12 release version as standard, and Python 3.13.0a2 as "future" test. + +### Dependencies + +* updated to black[jupyter]==23.12 (from black[jupyter]==23.11) +* updated to ruff==0.1.8 (from ruff==0.1.6) +* updated to pyright==1.1.338 (from pyright==1.1.336) + + ## [0.2.9] - 2023-09-20 ### Dependencies @@ -206,7 +221,8 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/e * Added support for Python 3.10 -[unreleased]: https://github.com/dnv-opensource/dictIO/compare/v0.2.9...HEAD +[unreleased]: https://github.com/dnv-opensource/dictIO/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/dnv-opensource/dictIO/compare/v0.2.9...v0.3.0 [0.2.9]: https://github.com/dnv-opensource/dictIO/compare/v0.2.8...v0.2.9 [0.2.8]: https://github.com/dnv-opensource/dictIO/compare/v0.2.7...v0.2.8 [0.2.7]: https://github.com/dnv-opensource/dictIO/compare/v0.2.6...v0.2.7 From beaad103d7c7b09815cc3fdf4cd98a389e2fea25 Mon Sep 17 00:00:00 2001 From: Claas Date: Fri, 22 Dec 2023 09:58:09 +0100 Subject: [PATCH 12/18] bumped version number to 0.3.0 --- docs/source/conf.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 81c792c8..bce900cf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -22,7 +22,7 @@ author = "Frank Lumpitzsch, Claas Rostock, Seung Hyeon Yoo" # The full version, including alpha/beta/rc tags -release = "0.2.9" +release = "0.3.0" # -- General configuration --------------------------------------------------- diff --git a/setup.cfg b/setup.cfg index 77929844..76027755 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dictIO -version = 0.2.9 +version = 0.3.0 summary = Read, write and manipulate dictionary text files. description = Python package to read, write and manipulate dictionary text files. Supports dictIOs dict file format, as well as JSON, XML and OpenFOAM. long_description = file: README.md From eeea5c76735fe977febe9fb318a145bc825da70d Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 27 Dec 2023 19:11:22 +0100 Subject: [PATCH 13/18] requirements-dev.txt: removed pytest-randomly (I don't think we need it) --- requirements-dev.txt | 1 - setup.cfg | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index facdf688..a886cb94 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ pytest>=7.4 pytest-cov>=4.1 -pytest-randomly>=3.15 black[jupyter]==23.12 ruff==0.1.8 pyright==1.1.338 diff --git a/setup.cfg b/setup.cfg index 77929844..71431055 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,6 +82,5 @@ system_site_packages = True deps = pytest>=7.4 pytest-cov>=4.1 - pytest-randomly>=3.15 commands = pytest --cov --cov-config setup.cfg {posargs} From 2648e3402e85bcb04d970358e0a4a130874dcfe7 Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 27 Dec 2023 19:12:58 +0100 Subject: [PATCH 14/18] pyproject.toml: removed pythonpath entry from pytest.ini_options (not needed as long as a .env file is used) --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c15f6402..c65209a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,4 +111,3 @@ reportUntypedNamedTuple = "warning" testpaths = "tests" addopts = "--strict-markers" xfail_strict = true -pythonpath = ["src"] From 57769b56f4a8ab0dd66f94a865782f6180f09191 Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 27 Dec 2023 19:14:04 +0100 Subject: [PATCH 15/18] setup.cfg: removed pytest options as these have been moved to pyproject.toml --- setup.cfg | 5 ----- 1 file changed, 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 71431055..2abf4e57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,11 +52,6 @@ exclude = console_scripts = dictParser = dictIO.cli.dictParser:main -[tool:pytest] -testpaths = tests -addopts = --strict-markers -xfail_strict = True - [coverage:run] source = dictIO branch = True From 702225f5f9445a76bfe1733084cf333cbe045eb8 Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 27 Dec 2023 19:15:31 +0100 Subject: [PATCH 16/18] VS Code debug launch configurations: updated the launch configuration for debugging tests --- .vscode/launch.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 6b91b5b0..76e6c876 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,11 +5,21 @@ "version": "0.2.0", "configurations": [ { - "name": "Debug Unit Test", + "name": "Python: Debug Tests", "type": "python", "request": "launch", + "program": "${file}", + "purpose": [ + "debug-test" + ], + "console": "integratedTerminal", + "env": { + "PYTEST_ADDOPTS": "--no-cov" + }, + "autoReload": { + "enable": true + }, "justMyCode": false, - "program": "${file}" }, { "name": "Python: Current File, cwd = file dir, envFile", From 36aa4f592ad8c2f360f3edf3ce1b84104bd7bb11 Mon Sep 17 00:00:00 2001 From: Claas Date: Wed, 27 Dec 2023 19:17:12 +0100 Subject: [PATCH 17/18] VS Code settings: deleted orphaned entries --- .vscode/settings.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 52762a52..398edb90 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,8 +5,6 @@ "python.terminal.activateEnvInCurrentTerminal": true, "python.languageServer": "Pylance", "ruff.importStrategy": "fromEnvironment", - "python.linting.enabled": false, - "python.formatting.provider": "black", "editor.formatOnSave": true, "[python]": { "editor.formatOnSave": true, @@ -15,6 +13,7 @@ }, }, "autoDocstring.docstringFormat": "numpy", + "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.analysis.logLevel": "Warning", "python.analysis.completeFunctionParens": false, @@ -22,9 +21,8 @@ "python.analysis.diagnosticSeverityOverrides": {}, "python.analysis.indexing": true, "python.analysis.autoImportCompletions": true, - "python.analysis.autoImportUserSymbols": true, "python.analysis.inlayHints.variableTypes": false, "python.analysis.inlayHints.functionReturnTypes": false, "python.analysis.inlayHints.pytestParameters": true, "python.terminal.executeInFileDir": true, -} +} \ No newline at end of file From fb266f2c2f84f04cb2e6a643c3182d8051782a74 Mon Sep 17 00:00:00 2001 From: Claas Date: Mon, 8 Jan 2024 17:39:48 +0100 Subject: [PATCH 18/18] updated CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b9ba030..9252eaee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/e * -/- -## [0.3.0] - 2023-12-22 +## [0.3.0] - 2024-01-08 ### Changed