From 91e0be6b7d728532a4fc163423c2934ab87d7104 Mon Sep 17 00:00:00 2001 From: Stephanie Kemna <6518317+StephanieKemna@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:04:21 +0200 Subject: [PATCH] Update line endings given new .gitattributes, add PR changes to CHANGELOG --- .github/pull_request_template.md | 96 ++-- CHANGELOG.md | 416 +++++++------- README.md | 524 +++++++++--------- docs/source/conf.py | 150 ++--- requirements-dev.txt | 22 +- requirements.txt | 18 +- tests/conftest.py | 106 ++-- tests/test_api.py | 2 +- .../test_config_file.json | 4 +- 9 files changed, 669 insertions(+), 669 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d17e461..76b6426 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,48 +1,48 @@ -## Description - - - - - - -Resolves #issue_nr - - - -... - - - -## How Has This Been Tested? - - - - -- [ ] Test A -- [ ] Test B - -## Screenshots (if appropriate) - -## Developer Checklist (before requesting PR review) - -- [ ] My code follows the style guidelines of this project -- [ ] My changes generate no new warnings -- [ ] Any dependent changes have been merged and published in downstream modules - -I have: - -- [ ] commented my code, particularly in hard-to-understand areas -- [ ] performed a self-review of my own code -- [ ] not committed unnecessary formatting changes, thereby occluding the actual changes (e.g. change of tab spacing, eol, etc.) -- [ ] made corresponding changes to the documentation -- [ ] added change to CHANGELOG.md -- [ ] added tests that prove my fix is effective or that my feature works (for core features) - -## Reviewer checklist - -I have: - -- [ ] performed a self-review of my own code have performed a review of the code -- [ ] tested that the software still works as expected -- [ ] checked updates to documentation -- [ ] checked that the CHANGELOG is updated +## Description + + + + + + +Resolves #issue_nr + + + +... + + + +## How Has This Been Tested? + + + + +- [ ] Test A +- [ ] Test B + +## Screenshots (if appropriate) + +## Developer Checklist (before requesting PR review) + +- [ ] My code follows the style guidelines of this project +- [ ] My changes generate no new warnings +- [ ] Any dependent changes have been merged and published in downstream modules + +I have: + +- [ ] commented my code, particularly in hard-to-understand areas +- [ ] performed a self-review of my own code +- [ ] not committed unnecessary formatting changes, thereby occluding the actual changes (e.g. change of tab spacing, eol, etc.) +- [ ] made corresponding changes to the documentation +- [ ] added change to CHANGELOG.md +- [ ] added tests that prove my fix is effective or that my feature works (for core features) + +## Reviewer checklist + +I have: + +- [ ] performed a self-review of my own code have performed a review of the code +- [ ] tested that the software still works as expected +- [ ] checked updates to documentation +- [ ] checked that the CHANGELOG is updated diff --git a/CHANGELOG.md b/CHANGELOG.md index 7682d73..cd8cc24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,208 +1,208 @@ -# Changelog - -All notable changes to the [mlfmu] project will be documented in this file.
-The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - -## [Unreleased] - -### Changed - -* Moved CMake + conan + c++ package files and folders with cpp code inside src folder to be included in package -* Replace pkg_resources with importlib.metadata for getting packages version to work in python 3.12 -* Replace deprecated root_validator with model_validator in pydanitc class -* Remove unnecessary hard coded values in utils/builder.py -* Complete cli interface - * Add subparsers to cli argparser to handle several different commands - * Create MlFmuBuilder from args and run code according to command -* Change cli test to match the new cli interface/parser -* Default agent_(input/output)_indexes is [] by default instead of None -* Updated doc by running publish-interface-docs -* Deleted azure files from old azure devops repo -* The generated modelDescription.xml files now also contain with a list of the outputs of the FMU. -* OnnxFmu cpp template class updated to be able to initialize state with FMU Variables - * Add variables to specify which FMU variable that should be used to initialize which state - * Add function to DoStep that initializes the state at the beginning of the first time step. -* model_definitions_template.h - * Add definitions for the the number of states that should be initialized and array of index/value reference pairs to describe how the states should be initialized -* Fmu Component json interface - * Add name, description and start_value to state in the json interface - * States changed from a single InternalState to a list of InternalState - -* Fixed typo from 'tunnable' to 'tunable' -* Fixed number of onnx output check to be correct (1-3 and not always raising exception) -* Fix correct fmi causality for parameters -* Fix correct default variability for parameters -* Fix expanding of array variables into one variable per index for inputs and parameters to work as outputs -* Default build target in builder to wind_to_power example -* CMakeList.txt (from old repo) to be able to take an arbitrary path where the fmu files to be compiled are located -* Builder to run commands to compile generated files into an fmu - * Can take and keep track of arbitrary paths -* Builder checks if the files needed to compile the fmu exists before trying to compile the fmu -* Replaced "TODO: Trow Error?" with raising a fitting exception -* replaced black formatter with ruff formatter -* Changed publishing workflow to use OpenID Connect (Trusted Publisher Management) when publishing to PyPI -* Updated copyright statement -* VS Code settings: Turned off automatic venv activation - -### Added - -* Add .github/pull_request_template.md for enabling PR templates on Github -* Add conan dependency to pyproject.toml -* Add MlFmuBuilder class to generate code and compile FMU - * Find default paths to files and directories if not given in cli - * Run functions in utils/builder.py according to which command is being run - * Clean up temporary/build files after the process is done -* Added feature to be able to initialize states using previously defined parameters or inputs - * This is done by setting "initializationVariable" = "{variable name}", instead of using the "name" and "start_value" attributes - * Added flag in schema to allow a variable to be reused when initializing states - * Allowed for multiple states to use the same variable for initialization -* .clang-format to consistently format cpp code -* Added code to generate parameters for initialization of state -* Wind to power model example in examples - * Onnx file containing ml model - * Interface json file containing information needed not contained in onnx file -* The generated fmu files resulting from running running builder on the new wind_to_power example -* CMakeLists.txt and conanfile.txt from old repo to configure compiling/building FMU from generated files -* README.md : Under `Development Setup`, added a step to install current package in "editable" mode, using the pip install -e option. -This removes the need to manually add /src to the PythonPath environment variable in order for debugging and tests to work. - -### Dependencies - -* Updated to ruff==0.5.1 (from ruff==0.4.2) -* Updated to pyright==1.1.371 (from pyright==1.1.360) -* Updated to sourcery==1.21 (from sourcery==1.16) -* Updated to dictIO>=0.3.4 (from dictIO>=0.3.4) -* Updated to sphinx-argparse-cli>=1.16 (from sphinx-argparse-cli>=1.15) -* Updated to furo>=2024.5 (from furo>=2024.4) -* Updated to setup-python@v5 (from setup-python@v4) -* Updated to actions-gh-pages@v4 (from actions-gh-pages@v3) -* Updated to upload-artifact@v4 (from upload-artifact@v3) -* GitHub workflows: Replaced pip install tox with pip install tox-uv -* GitHub workflows: Removed cache: 'pip' for tox-uv compatibility -* GitHub workflows: Install dependencies: change singleline run statements to multiline run statements -* GitHub workflows: Add step to install 'uv' package -* GitHub workflows: Add step to install 'uv' package -* GitHub workflows: Install dependencies: change from 'pip install' to 'uv pip install' -* GitHub workflow _test_future.yml : updated Python version to 3.13.0-alpha - 3.13.0 -* GitHub workflow _test_future.yml : updated name of test job to 'test313' -* updated to ruff==0.4.2 (from ruff==0.3.0) -* updated to pyright==1.1.360 (from pyright==1.1.352) -* updated to sourcery==1.16 (from sourcery==1.15) -* updated to pytest>=8.2 (from pytest>=7.4) -* updated to pytest-cov>=5.0 (from pytest-cov>=4.1) -* updated to Sphinx>=7.3 (from Sphinx>=7.2) -* updated to sphinx-argparse-cli>=1.15 (from sphinx-argparse-cli>=1.11) -* updated to myst-parser>=3.0 (from myst-parser>=2.0) -* updated to furo>=2024.4 (from furo>=2023.9.10) -* updated to ruff==0.3.0 (from ruff==0.2.1) -* updated to pyright==1.1.352 (from pyright==1.1.350) -* removed black -* updated to dictIO>=0.3.3 (from dictIO>=0.3.1) - -### Removed - -* VS Code settings: Removed the setting which added the /src folder to PythonPath. This is no longer necessary. Installing the project itself as a package in "editable" mode, using the pip install -e option, solves the issue and removes the need to manually add /src to the PythonPath environment variable. - -## [0.1.6] - 2024-02-20 - -### Changed - -* Moved all project configuration from setup.cfg to pyproject.toml -* Moved all tox configuration from setup.cfg to tox.ini. -* Moved pytest configuration from pyproject.toml to pytest.ini -* Deleted setup.cfg - -### Dependencies - -* updated to black[jupyter]==24.1 (from black[jupyter]==23.12) -* updated to ruff==0.2.1 (from ruff==0.1.8) -* updated to pyright==1.1.350 (from pyright==1.1.338) -* updated to sourcery==1.15 (from sourcery==1.14) -* updated to dictIO>=0.3.1 (from dictIO>=0.2.9) - -## [0.1.5] - 2023-11-08 - -### Changed - -* incorporated latest updates introduced in mvx - -## [0.1.4] - 2023-09-25 - -### Dependencies - -* Updated dependencies to latest versions - -## [0.1.3] - 2023-08-24 - -### Changed - -* GitHub workflow publish_release.yml: corrected smaller errors -* Explicitly removed .env file from remote repository -* Updated README.md to include guidance on how to create a .env file locally -* dependencies: updated packages in requirements-dev.txt to latest versions - -## [0.1.2] - 2023-06-22 - -### Changed - -* Modularized GitHub workflows -* requirements-dev.txt: Updated dependencies to latest versions -* setup.cfg: indicated supported Python versions as py310 and py311
- (from formerly py39 and py310) -* GitHub workflows: changed default Python version from 3.10 to 3.11 - -## [0.1.1] - 2023-05-02 - -### Changed - -* requirements-dev.txt: Updated dependencies to latest versions - -## [0.1.0] - 2023-02-21 - -### Changed - -* pyproject.toml: Changed ruff configuration to by default allow Uppercase variable names in functions.
-(As this is a very common case in science calculus) -* README.md: Changed install infos for development setup to pip install requirements-dev.txt (not requirements.txt) - -## [0.0.1] - 2023-02-21 - -* Initial release - -### Added - -* added this - -### Changed - -* changed that - -### Dependencies - -* updated to some_package_on_pypi>=0.1.0 - -### Fixed - -* fixed issue #12345 - -### Deprecated - -* following features will soon be removed and have been marked as deprecated: - * function x in module z - -### Removed - -* following features have been removed: - * function y in module z - - -[unreleased]: https://github.com/dnv-innersource/mlfmu/compare/v0.1.6...HEAD -[0.1.6]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.5...v0.1.6 -[0.1.5]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.4...v0.1.5 -[0.1.4]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.3...v0.1.4 -[0.1.3]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.2...v0.1.3 -[0.1.2]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.1...v0.1.2 -[0.1.1]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.0...v0.1.1 -[0.1.0]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.0.1...v0.1.0 -[0.0.1]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.0.1 -[mlfmu]: https://github.com/dnv-innersource/mlfmu +# Changelog + +All notable changes to the [mlfmu] project will be documented in this file.
+The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [Unreleased] + +### Changed + +* Moved CMake + conan + c++ package files and folders with cpp code inside src folder to be included in package +* Replace pkg_resources with importlib.metadata for getting packages version to work in python 3.12 +* Replace deprecated root_validator with model_validator in pydanitc class +* Remove unnecessary hard coded values in utils/builder.py +* Complete cli interface + * Add subparsers to cli argparser to handle several different commands + * Create MlFmuBuilder from args and run code according to command +* Change cli test to match the new cli interface/parser +* Default agent_(input/output)_indexes is [] by default instead of None +* Updated doc by running publish-interface-docs +* Deleted azure files from old azure devops repo +* The generated modelDescription.xml files now also contain with a list of the outputs of the FMU. +* OnnxFmu cpp template class updated to be able to initialize state with FMU Variables + * Add variables to specify which FMU variable that should be used to initialize which state + * Add function to DoStep that initializes the state at the beginning of the first time step. +* model_definitions_template.h + * Add definitions for the the number of states that should be initialized and array of index/value reference pairs to describe how the states should be initialized +* Fmu Component json interface + * Add name, description and start_value to state in the json interface + * States changed from a single InternalState to a list of InternalState +* Fixed typo from 'tunnable' to 'tunable' +* Fixed number of onnx output check to be correct (1-3 and not always raising exception) +* Fix correct fmi causality for parameters +* Fix correct default variability for parameters +* Fix expanding of array variables into one variable per index for inputs and parameters to work as outputs +* Default build target in builder to wind_to_power example +* CMakeList.txt (from old repo) to be able to take an arbitrary path where the fmu files to be compiled are located +* Builder to run commands to compile generated files into an fmu + * Can take and keep track of arbitrary paths +* Builder checks if the files needed to compile the fmu exists before trying to compile the fmu +* Replaced "TODO: Trow Error?" with raising a fitting exception +* replaced black formatter with ruff formatter +* Changed publishing workflow to use OpenID Connect (Trusted Publisher Management) when publishing to PyPI +* Updated copyright statement +* VS Code settings: Turned off automatic venv activation + +### Added + +* Add .gitattributes to handle line endings, removed eol from .editorconfig +* Add .github/pull_request_template.md for enabling PR templates on Github +* Add conan dependency to pyproject.toml +* Add MlFmuBuilder class to generate code and compile FMU + * Find default paths to files and directories if not given in cli + * Run functions in utils/builder.py according to which command is being run + * Clean up temporary/build files after the process is done +* Added feature to be able to initialize states using previously defined parameters or inputs + * This is done by setting "initializationVariable" = "{variable name}", instead of using the "name" and "start_value" attributes + * Added flag in schema to allow a variable to be reused when initializing states + * Allowed for multiple states to use the same variable for initialization +* .clang-format to consistently format cpp code +* Added code to generate parameters for initialization of state +* Wind to power model example in examples + * Onnx file containing ml model + * Interface json file containing information needed not contained in onnx file +* The generated fmu files resulting from running running builder on the new wind_to_power example +* CMakeLists.txt and conanfile.txt from old repo to configure compiling/building FMU from generated files +* README.md : Under `Development Setup`, added a step to install current package in "editable" mode, using the pip install -e option. +This removes the need to manually add /src to the PythonPath environment variable in order for debugging and tests to work. + +### Dependencies + +* Updated to ruff==0.5.1 (from ruff==0.4.2) +* Updated to pyright==1.1.371 (from pyright==1.1.360) +* Updated to sourcery==1.21 (from sourcery==1.16) +* Updated to dictIO>=0.3.4 (from dictIO>=0.3.4) +* Updated to sphinx-argparse-cli>=1.16 (from sphinx-argparse-cli>=1.15) +* Updated to furo>=2024.5 (from furo>=2024.4) +* Updated to setup-python@v5 (from setup-python@v4) +* Updated to actions-gh-pages@v4 (from actions-gh-pages@v3) +* Updated to upload-artifact@v4 (from upload-artifact@v3) +* GitHub workflows: Replaced pip install tox with pip install tox-uv +* GitHub workflows: Removed cache: 'pip' for tox-uv compatibility +* GitHub workflows: Install dependencies: change singleline run statements to multiline run statements +* GitHub workflows: Add step to install 'uv' package +* GitHub workflows: Add step to install 'uv' package +* GitHub workflows: Install dependencies: change from 'pip install' to 'uv pip install' +* GitHub workflow _test_future.yml : updated Python version to 3.13.0-alpha - 3.13.0 +* GitHub workflow _test_future.yml : updated name of test job to 'test313' +* updated to ruff==0.4.2 (from ruff==0.3.0) +* updated to pyright==1.1.360 (from pyright==1.1.352) +* updated to sourcery==1.16 (from sourcery==1.15) +* updated to pytest>=8.2 (from pytest>=7.4) +* updated to pytest-cov>=5.0 (from pytest-cov>=4.1) +* updated to Sphinx>=7.3 (from Sphinx>=7.2) +* updated to sphinx-argparse-cli>=1.15 (from sphinx-argparse-cli>=1.11) +* updated to myst-parser>=3.0 (from myst-parser>=2.0) +* updated to furo>=2024.4 (from furo>=2023.9.10) +* updated to ruff==0.3.0 (from ruff==0.2.1) +* updated to pyright==1.1.352 (from pyright==1.1.350) +* removed black +* updated to dictIO>=0.3.3 (from dictIO>=0.3.1) + +### Removed + +* VS Code settings: Removed the setting which added the /src folder to PythonPath. This is no longer necessary. Installing the project itself as a package in "editable" mode, using the pip install -e option, solves the issue and removes the need to manually add /src to the PythonPath environment variable. + +## [0.1.6] - 2024-02-20 + +### Changed + +* Moved all project configuration from setup.cfg to pyproject.toml +* Moved all tox configuration from setup.cfg to tox.ini. +* Moved pytest configuration from pyproject.toml to pytest.ini +* Deleted setup.cfg + +### Dependencies + +* updated to black[jupyter]==24.1 (from black[jupyter]==23.12) +* updated to ruff==0.2.1 (from ruff==0.1.8) +* updated to pyright==1.1.350 (from pyright==1.1.338) +* updated to sourcery==1.15 (from sourcery==1.14) +* updated to dictIO>=0.3.1 (from dictIO>=0.2.9) + +## [0.1.5] - 2023-11-08 + +### Changed + +* incorporated latest updates introduced in mvx + +## [0.1.4] - 2023-09-25 + +### Dependencies + +* Updated dependencies to latest versions + +## [0.1.3] - 2023-08-24 + +### Changed + +* GitHub workflow publish_release.yml: corrected smaller errors +* Explicitly removed .env file from remote repository +* Updated README.md to include guidance on how to create a .env file locally +* dependencies: updated packages in requirements-dev.txt to latest versions + +## [0.1.2] - 2023-06-22 + +### Changed + +* Modularized GitHub workflows +* requirements-dev.txt: Updated dependencies to latest versions +* setup.cfg: indicated supported Python versions as py310 and py311
+ (from formerly py39 and py310) +* GitHub workflows: changed default Python version from 3.10 to 3.11 + +## [0.1.1] - 2023-05-02 + +### Changed + +* requirements-dev.txt: Updated dependencies to latest versions + +## [0.1.0] - 2023-02-21 + +### Changed + +* pyproject.toml: Changed ruff configuration to by default allow Uppercase variable names in functions.
+(As this is a very common case in science calculus) +* README.md: Changed install infos for development setup to pip install requirements-dev.txt (not requirements.txt) + +## [0.0.1] - 2023-02-21 + +* Initial release + +### Added + +* added this + +### Changed + +* changed that + +### Dependencies + +* updated to some_package_on_pypi>=0.1.0 + +### Fixed + +* fixed issue #12345 + +### Deprecated + +* following features will soon be removed and have been marked as deprecated: + * function x in module z + +### Removed + +* following features have been removed: + * function y in module z + + +[unreleased]: https://github.com/dnv-innersource/mlfmu/compare/v0.1.6...HEAD +[0.1.6]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.5...v0.1.6 +[0.1.5]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.4...v0.1.5 +[0.1.4]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.3...v0.1.4 +[0.1.3]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.2...v0.1.3 +[0.1.2]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.1...v0.1.2 +[0.1.1]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.0.1...v0.1.0 +[0.0.1]: https://github.com/dnv-innersource/mlfmu/releases/tag/v0.0.1 +[mlfmu]: https://github.com/dnv-innersource/mlfmu diff --git a/README.md b/README.md index ecded86..bfb606c 100644 --- a/README.md +++ b/README.md @@ -1,262 +1,262 @@ -# mlfmu - -MLFMU serves as a tool for developers looking to integrate machine learning models into simulation environments. It enables the creation of Functional Mock-up Units (FMUs), which are simulation models that adhere to the FMI standard (), from trained machine learning models exported in the ONNX format (). The mlfmu package streamlines the process of transforming ONNX models into FMUs, facilitating their use in a wide range of simulation platforms that support the FMI standard such as the [Open Simulation Platform](https://open-simulation-platform.github.io/) or DNV's [Simulation Trust Center](https://store.veracity.com/simulation-trust-center) - -## Features - -- Compile trained ML models into FMUs (Functional Mock-up Units). -- Easy to integrate in building pipelines. -- Declarative solution, just define what the inputs/outputs/parameters of your co-simulation model should look like and MLFMU will take care of the rest. -- Support for FMU signal vectors in FMI 2.0. -- Advanced customizations by enabling you to change the C++ code of the FMU. - -## Installation - -```sh -pip install mlfmu -``` - -## Creating ML FMUs - -### Create your own ML model - -Before you use this mlfmu tool, you should create your machine learning (ML) model, using whatever your preferred tool is. - -1. Define the architecture of your ML model and prepare the model to receive the inputs following to MLFMU's input format. - -> Note 1: This example subclasses a Keras model for demonstration purposes. However, the tool is flexible and can accommodate other frameworks such as PyTorch, TensorFlow, Scikit-learn, and more. - -> Note 2: We showcase a simple example here. For more detailed information on how you can prepare your model to be compatible with this tool, see [MLMODEL.md](MLMODEL.md) - -```python -# Create your ML model -class MlModel(tf.keras.Model): - def init(self, num_inputs = 2): - # 1 hidden layer, 1 output layer - self.hidden_layer = tf.keras.layers.Dense(512, activation=tf.nn.relu) - self.output_layer = tf.keras.layers.Dense(1, activation=None) - - ... - - def call(self, all_inputs): # model forward pass - # unpack inputs - inputs, *_ = all_inputs - - # Do something with the inputs - # Here we have 1 hidden layer - d1 = self.hidden_layer(inputs) - outputs = self.output_layer(d1) - - return outputs - ... -``` - -2. Train your model, then save it as an ONNX file, e.g.: - -```python -import onnx - -ml_model = MlModel() -# compile: configure model for training -ml_model.compile(optimizer=tf.optimizers.RMSProp, loss='mse') -# fit: train your ML model for some number of epochs -ml_model.fit(training_dataset, epochs=nr_epochs) - -# Save the trained model as ONNX at a specified path -onnx_model = tf2onnx.convert.from_keras(ml_model) -onnx.save(onnx_model, 'path/to/save') -``` - -3. (Optional) You may want to check your onnx file to make sure it produces the right output. You can do this by loading the onnx file and (using the same test input) compare the onnx model predictions to your original model predictions. -You can also check the model using Netron: or - -### Preparing for and using MLFMU - -Given that you have an ML model, you now need to: - -1. Prepare the FMU interface specification (.json), to specify your FMU's inputs, parameters, and output, map these to the ML model's inputs and output (`agentInputIndexes`) and to specify whether it uses time (`usesTime`). - -```json -// Interface.json -{ - "name": "MyMLFMU", - "description": "A Machine Learning based FMU", - "usesTime": true, - "inputs": [ - { - "name": "input_1", - "description": "My input signal to the model at position 0", - "agentInputIndexes": ["0"] - } - ], - "parameters": [ - { - "name": "parameter_1", - "description": "My input signal to the model at position 1", - "agentInputIndexes": ["1"] - } - ], - "outputs": [ - { - "name": "prediction", - "description": "The prediction generated by ML model", - "agentOutputIndexes": ["0"] - } - ] -} -``` - -2. Compile the FMU: - -```sh -mlfmu build --interface-file Interface.json --model-file model.onnx -``` - -or if the files are in your current working directory: - -```sh -mlfmu build -``` - -# Extended documentation - -For more explanation on the ONNX file structure and inputs/outputs for your model, please refer to mlfmu's [MLMODEL.md](MLMODEL.md). - -For advanced usage options, e.g. editing the generated FMU source code, or using the tool via a Python class, please refer to mlfmu's [ADVANCED.md](ADVANCED.md). - -## Development Setup - -1. Install Python 3.9 or higher, i.e. [Python 3.10](https://www.python.org/downloads/release/python-3104/) or [Python 3.11](https://www.python.org/downloads/release/python-3114/) - -2. Update pip and setuptools: - - ```sh - python -m pip install --upgrade pip setuptools - ``` - -3. git clone the mlfmu repository into your local development directory: - - ```sh - git clone https://github.com/dnv-innersource/mlfmu path/to/your/dev/mlfmu - git submodule update --init --recursive - ``` - -4. In the mlfmu root folder: - - Create a Python virtual environment, e.g. (you can also make a conda environment): - - ```sh - python -m venv .venv - ``` - - Activate the virtual environment: - - ..on Windows: - - ```sh - > .venv\Scripts\activate.bat - ``` - - ..on Linux: - - ```sh - source .venv/bin/activate - ``` - - Install/update pip and setuptools: - - ```sh - (.venv) $ python -m pip install --upgrade pip setuptools - ``` - - (Optional) If you want PyTorch cuda support on your local machine - (i.e. to use your GPU for torch operations), you should preferably install PyTorch with cuda support first, before installing all other dependendencies. - On the official PyTorch website at , - you can generate a pip install command matching your local machine's operating system, using a wizard. - If you are on Windows, the resulting pip install command will most likely look something like this: - - ```sh - (.venv) $ pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 - ``` - - _Hint:_ If you are unsure which cuda version to indicate in above `pip install .. /cuXXX` command, you can use the shell command `nvidia-smi` on your local system to find out the cuda version supported by the current graphics driver installed on your system. When then generating the `pip install` command with the wizard from , select the cuda version that matches the major version of what your graphics driver supports (major version must match, minor version may deviate). - - > Note: We use conan for building the FMU. For the conan building to work later on, you will need the Visual Studio Build tools 2022 to be installed. It is best to do this **before** installing conan (which we install via pip install of requirements). You can download and install the Build Tools for VS 2022 (for free) from . - - > Note 2: After you install conan, you want to make sure it has the correct build profile. You can auto-detect and create the profile by running `conan profile detect`. After this, you can check the profile in `C:\Users\\.conan2\profiles\.default` (replace `` with your username). You want to `compiler=msvc`, `compiler.cppstd=17`, `compiler.version=193` (for Windows). - - Install mlfmu's dependencies.
- - ```sh - (.venv) $ pip install -r requirements-dev.txt - ``` - - This should return without errors. - - Finally, install mlfmu itself, yet not as a regular package but as an _editable_ package instead, using the pip install option -e: - - ```sh - (.venv) $ pip install -e . - ``` - -5. Test that the installation works (in the mlfmu root folder): - - ```sh - (.venv) $ pytest . - ``` - -6. Run an example: - - ```sh - (.venv) $ cd .\examples\wind_generator\config\ - (.venv) $ mlfmu build - ``` - -As an alternative, you can run from the main directory: - -```sh -mlfmu build --interface-file .\examples\wind_generator\config\interface.json --model-file .\examples\wind_generator\config\example.onnx -``` - -Note; wherever you run the build command from, is where the FMU file will be created, unless you specify otherwise with `--fmu-path`. - -For more options, see `mlfmu --help` or `mlfmu build --help`. - -## Meta - -All code in mlfmu is DNV intellectual property and for DNV internal use only. - -Copyright (c) 2024 [DNV](https://www.dnv.com) AS. All rights reserved. - -Kristoffer Skare - [@LinkedIn](https://www.linkedin.com/in/kristoffer-skare-19606a1a1/) - - -Jorge Luis Mendez - [@LinkedIn](https://www.linkedin.com/in/jorgelmh/) - - -Stephanie Kemna - [@LinkedIn](https://www.linkedin.com/in/stephaniekemna/) - - -Author 4 - [@LinkedIn](https://www.linkedin.com/in/name/) - - -```diff -- TODO: (1) Adapt to chosen license (or delete if no license is applied).
-- TODO: (2) Adapt or delete the license file (LICENSE.md)
-- TODO: (3) Adapt or delete the license entry in setup.cfg
-Distributed under the XYZ license. See [LICENSE](LICENSE.md) for more information. -``` - -## Contributing - -1. Fork it () (Note: this is currently disabled for this repo. For DNV internal development, continue with the next step.) -2. Create an issue in your GitHub repo -3. Create your branch based on the issue number and type (`git checkout -b issue-name`) -4. Evaluate and stage the changes you want to commit (`git add -i`) -5. Commit your changes (`git commit -am 'place a descriptive commit message here'`) -6. Push to the branch (`git push origin issue-name`) -7. Create a new Pull Request in GitHub - -For your contribution, please make sure you follow the [STYLEGUIDE](STYLEGUIDE.md) before creating the Pull Request. - -## Errors & fixes - -- If you get an error similar to `..\fmu.cpp(4,10): error C1083: Cannot open include file: 'cppfmu_cs.hpp': No such file or directory`, you are missing cppfmu. This is a submodule to this repository. Make sure that you do a `git submodule update --init --recursive` in the top level folder. - - +# mlfmu + +MLFMU serves as a tool for developers looking to integrate machine learning models into simulation environments. It enables the creation of Functional Mock-up Units (FMUs), which are simulation models that adhere to the FMI standard (), from trained machine learning models exported in the ONNX format (). The mlfmu package streamlines the process of transforming ONNX models into FMUs, facilitating their use in a wide range of simulation platforms that support the FMI standard such as the [Open Simulation Platform](https://open-simulation-platform.github.io/) or DNV's [Simulation Trust Center](https://store.veracity.com/simulation-trust-center) + +## Features + +- Compile trained ML models into FMUs (Functional Mock-up Units). +- Easy to integrate in building pipelines. +- Declarative solution, just define what the inputs/outputs/parameters of your co-simulation model should look like and MLFMU will take care of the rest. +- Support for FMU signal vectors in FMI 2.0. +- Advanced customizations by enabling you to change the C++ code of the FMU. + +## Installation + +```sh +pip install mlfmu +``` + +## Creating ML FMUs + +### Create your own ML model + +Before you use this mlfmu tool, you should create your machine learning (ML) model, using whatever your preferred tool is. + +1. Define the architecture of your ML model and prepare the model to receive the inputs following to MLFMU's input format. + +> Note 1: This example subclasses a Keras model for demonstration purposes. However, the tool is flexible and can accommodate other frameworks such as PyTorch, TensorFlow, Scikit-learn, and more. + +> Note 2: We showcase a simple example here. For more detailed information on how you can prepare your model to be compatible with this tool, see [MLMODEL.md](MLMODEL.md) + +```python +# Create your ML model +class MlModel(tf.keras.Model): + def init(self, num_inputs = 2): + # 1 hidden layer, 1 output layer + self.hidden_layer = tf.keras.layers.Dense(512, activation=tf.nn.relu) + self.output_layer = tf.keras.layers.Dense(1, activation=None) + + ... + + def call(self, all_inputs): # model forward pass + # unpack inputs + inputs, *_ = all_inputs + + # Do something with the inputs + # Here we have 1 hidden layer + d1 = self.hidden_layer(inputs) + outputs = self.output_layer(d1) + + return outputs + ... +``` + +2. Train your model, then save it as an ONNX file, e.g.: + +```python +import onnx + +ml_model = MlModel() +# compile: configure model for training +ml_model.compile(optimizer=tf.optimizers.RMSProp, loss='mse') +# fit: train your ML model for some number of epochs +ml_model.fit(training_dataset, epochs=nr_epochs) + +# Save the trained model as ONNX at a specified path +onnx_model = tf2onnx.convert.from_keras(ml_model) +onnx.save(onnx_model, 'path/to/save') +``` + +3. (Optional) You may want to check your onnx file to make sure it produces the right output. You can do this by loading the onnx file and (using the same test input) compare the onnx model predictions to your original model predictions. +You can also check the model using Netron: or + +### Preparing for and using MLFMU + +Given that you have an ML model, you now need to: + +1. Prepare the FMU interface specification (.json), to specify your FMU's inputs, parameters, and output, map these to the ML model's inputs and output (`agentInputIndexes`) and to specify whether it uses time (`usesTime`). + +```json +// Interface.json +{ + "name": "MyMLFMU", + "description": "A Machine Learning based FMU", + "usesTime": true, + "inputs": [ + { + "name": "input_1", + "description": "My input signal to the model at position 0", + "agentInputIndexes": ["0"] + } + ], + "parameters": [ + { + "name": "parameter_1", + "description": "My input signal to the model at position 1", + "agentInputIndexes": ["1"] + } + ], + "outputs": [ + { + "name": "prediction", + "description": "The prediction generated by ML model", + "agentOutputIndexes": ["0"] + } + ] +} +``` + +2. Compile the FMU: + +```sh +mlfmu build --interface-file Interface.json --model-file model.onnx +``` + +or if the files are in your current working directory: + +```sh +mlfmu build +``` + +# Extended documentation + +For more explanation on the ONNX file structure and inputs/outputs for your model, please refer to mlfmu's [MLMODEL.md](MLMODEL.md). + +For advanced usage options, e.g. editing the generated FMU source code, or using the tool via a Python class, please refer to mlfmu's [ADVANCED.md](ADVANCED.md). + +## Development Setup + +1. Install Python 3.9 or higher, i.e. [Python 3.10](https://www.python.org/downloads/release/python-3104/) or [Python 3.11](https://www.python.org/downloads/release/python-3114/) + +2. Update pip and setuptools: + + ```sh + python -m pip install --upgrade pip setuptools + ``` + +3. git clone the mlfmu repository into your local development directory: + + ```sh + git clone https://github.com/dnv-innersource/mlfmu path/to/your/dev/mlfmu + git submodule update --init --recursive + ``` + +4. In the mlfmu root folder: + + Create a Python virtual environment, e.g. (you can also make a conda environment): + + ```sh + python -m venv .venv + ``` + + Activate the virtual environment: + + ..on Windows: + + ```sh + > .venv\Scripts\activate.bat + ``` + + ..on Linux: + + ```sh + source .venv/bin/activate + ``` + + Install/update pip and setuptools: + + ```sh + (.venv) $ python -m pip install --upgrade pip setuptools + ``` + + (Optional) If you want PyTorch cuda support on your local machine + (i.e. to use your GPU for torch operations), you should preferably install PyTorch with cuda support first, before installing all other dependendencies. + On the official PyTorch website at , + you can generate a pip install command matching your local machine's operating system, using a wizard. + If you are on Windows, the resulting pip install command will most likely look something like this: + + ```sh + (.venv) $ pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 + ``` + + _Hint:_ If you are unsure which cuda version to indicate in above `pip install .. /cuXXX` command, you can use the shell command `nvidia-smi` on your local system to find out the cuda version supported by the current graphics driver installed on your system. When then generating the `pip install` command with the wizard from , select the cuda version that matches the major version of what your graphics driver supports (major version must match, minor version may deviate). + + > Note: We use conan for building the FMU. For the conan building to work later on, you will need the Visual Studio Build tools 2022 to be installed. It is best to do this **before** installing conan (which we install via pip install of requirements). You can download and install the Build Tools for VS 2022 (for free) from . + + > Note 2: After you install conan, you want to make sure it has the correct build profile. You can auto-detect and create the profile by running `conan profile detect`. After this, you can check the profile in `C:\Users\\.conan2\profiles\.default` (replace `` with your username). You want to `compiler=msvc`, `compiler.cppstd=17`, `compiler.version=193` (for Windows). + + Install mlfmu's dependencies.
+ + ```sh + (.venv) $ pip install -r requirements-dev.txt + ``` + + This should return without errors. + + Finally, install mlfmu itself, yet not as a regular package but as an _editable_ package instead, using the pip install option -e: + + ```sh + (.venv) $ pip install -e . + ``` + +5. Test that the installation works (in the mlfmu root folder): + + ```sh + (.venv) $ pytest . + ``` + +6. Run an example: + + ```sh + (.venv) $ cd .\examples\wind_generator\config\ + (.venv) $ mlfmu build + ``` + +As an alternative, you can run from the main directory: + +```sh +mlfmu build --interface-file .\examples\wind_generator\config\interface.json --model-file .\examples\wind_generator\config\example.onnx +``` + +Note; wherever you run the build command from, is where the FMU file will be created, unless you specify otherwise with `--fmu-path`. + +For more options, see `mlfmu --help` or `mlfmu build --help`. + +## Meta + +All code in mlfmu is DNV intellectual property and for DNV internal use only. + +Copyright (c) 2024 [DNV](https://www.dnv.com) AS. All rights reserved. + +Kristoffer Skare - [@LinkedIn](https://www.linkedin.com/in/kristoffer-skare-19606a1a1/) - + +Jorge Luis Mendez - [@LinkedIn](https://www.linkedin.com/in/jorgelmh/) - + +Stephanie Kemna - [@LinkedIn](https://www.linkedin.com/in/stephaniekemna/) - + +Author 4 - [@LinkedIn](https://www.linkedin.com/in/name/) - + +```diff +- TODO: (1) Adapt to chosen license (or delete if no license is applied).
+- TODO: (2) Adapt or delete the license file (LICENSE.md)
+- TODO: (3) Adapt or delete the license entry in setup.cfg
+Distributed under the XYZ license. See [LICENSE](LICENSE.md) for more information. +``` + +## Contributing + +1. Fork it () (Note: this is currently disabled for this repo. For DNV internal development, continue with the next step.) +2. Create an issue in your GitHub repo +3. Create your branch based on the issue number and type (`git checkout -b issue-name`) +4. Evaluate and stage the changes you want to commit (`git add -i`) +5. Commit your changes (`git commit -am 'place a descriptive commit message here'`) +6. Push to the branch (`git push origin issue-name`) +7. Create a new Pull Request in GitHub + +For your contribution, please make sure you follow the [STYLEGUIDE](STYLEGUIDE.md) before creating the Pull Request. + +## Errors & fixes + +- If you get an error similar to `..\fmu.cpp(4,10): error C1083: Cannot open include file: 'cppfmu_cs.hpp': No such file or directory`, you are missing cppfmu. This is a submodule to this repository. Make sure that you do a `git submodule update --init --recursive` in the top level folder. + + diff --git a/docs/source/conf.py b/docs/source/conf.py index 430ba08..7cb259d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,75 +1,75 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys - -sys.path.insert(0, os.path.abspath("../../src")) - - -# -- Project information ----------------------------------------------------- - -project = "mlfmu" -copyright = "2024, DNV AS. All rights reserved." -author = "Author One, Author Two, Author Three" - -# The full version, including alpha/beta/rc tags -release = "0.1.6" - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "myst_parser", - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx_argparse_cli", -] - -# The file extensions of source files. -source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", -} - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "furo" -html_logo = "_static/mlfmu.svg" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -autodoc_default_options = { - "member-order": "bysource", - "undoc-members": True, - "exclude-members": "__weakref__", -} -autodoc_preserve_defaults = True - -myst_heading_anchors = 3 +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +sys.path.insert(0, os.path.abspath("../../src")) + + +# -- Project information ----------------------------------------------------- + +project = "mlfmu" +copyright = "2024, DNV AS. All rights reserved." +author = "Author One, Author Two, Author Three" + +# The full version, including alpha/beta/rc tags +release = "0.1.6" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx_argparse_cli", +] + +# The file extensions of source files. +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" +html_logo = "_static/mlfmu.svg" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +autodoc_default_options = { + "member-order": "bysource", + "undoc-members": True, + "exclude-members": "__weakref__", +} +autodoc_preserve_defaults = True + +myst_heading_anchors = 3 diff --git a/requirements-dev.txt b/requirements-dev.txt index 0307405..e9a1180 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,11 +1,11 @@ -pytest>=8.2 -pytest-cov>=5.0 -ruff==0.5.1 -pyright==1.1.371 -Sphinx>=7.3 -sphinx-argparse-cli>=1.16 -myst-parser>=3.0 -furo>=2024.5 -sourcery==1.21 - --r requirements.txt +pytest>=8.2 +pytest-cov>=5.0 +ruff==0.5.1 +pyright==1.1.371 +Sphinx>=7.3 +sphinx-argparse-cli>=1.16 +myst-parser>=3.0 +furo>=2024.5 +sourcery==1.21 + +-r requirements.txt diff --git a/requirements.txt b/requirements.txt index 44d58bf..ebe050e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -dictIO>=0.3.4 -pydantic>=2.6 -json-schema-for-humans>=0.4.7 -onnxruntime==1.18.1 -conan>=2.1 - -# some_package_on_pypi>=0.1.0 - -# ../some_project_in_my_local_dev +dictIO>=0.3.4 +pydantic>=2.6 +json-schema-for-humans>=0.4.7 +onnxruntime==1.18.1 +conan>=2.1 + +# some_package_on_pypi>=0.1.0 + +# ../some_project_in_my_local_dev diff --git a/tests/conftest.py b/tests/conftest.py index 129f993..ec37c2e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,53 +1,53 @@ -import logging -import os -from glob import glob -from pathlib import Path -from shutil import rmtree - -import pytest -from pytest import LogCaptureFixture - - -@pytest.fixture(scope="package", autouse=True) -def chdir(): - os.chdir(Path(__file__).parent.absolute() / "test_working_directory") - - -@pytest.fixture(scope="package", autouse=True) -def test_dir(): - return Path(__file__).parent.absolute() - - -output_dirs = [ - "results", -] -output_files = [ - "*test*.pdf", -] - - -@pytest.fixture(autouse=True) -def default_setup_and_teardown(caplog: LogCaptureFixture): - _remove_output_dirs_and_files() - yield - _remove_output_dirs_and_files() - - -def _remove_output_dirs_and_files(): - for folder in output_dirs: - rmtree(folder, ignore_errors=True) - for pattern in output_files: - for file in glob(pattern): - file = Path(file) - file.unlink(missing_ok=True) - - -@pytest.fixture(autouse=True) -def setup_logging(caplog: LogCaptureFixture): - caplog.set_level("INFO") - caplog.clear() - - -@pytest.fixture(autouse=True) -def logger(): - return logging.getLogger() +import logging +import os +from glob import glob +from pathlib import Path +from shutil import rmtree + +import pytest +from pytest import LogCaptureFixture + + +@pytest.fixture(scope="package", autouse=True) +def chdir(): + os.chdir(Path(__file__).parent.absolute() / "test_working_directory") + + +@pytest.fixture(scope="package", autouse=True) +def test_dir(): + return Path(__file__).parent.absolute() + + +output_dirs = [ + "results", +] +output_files = [ + "*test*.pdf", +] + + +@pytest.fixture(autouse=True) +def default_setup_and_teardown(caplog: LogCaptureFixture): + _remove_output_dirs_and_files() + yield + _remove_output_dirs_and_files() + + +def _remove_output_dirs_and_files(): + for folder in output_dirs: + rmtree(folder, ignore_errors=True) + for pattern in output_files: + for file in glob(pattern): + file = Path(file) + file.unlink(missing_ok=True) + + +@pytest.fixture(autouse=True) +def setup_logging(caplog: LogCaptureFixture): + caplog.set_level("INFO") + caplog.clear() + + +@pytest.fixture(autouse=True) +def logger(): + return logging.getLogger() diff --git a/tests/test_api.py b/tests/test_api.py index 2687c98..86e2701 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1 +1 @@ -# pyright: reportPrivateUsage=false +# pyright: reportPrivateUsage=false diff --git a/tests/test_working_directory/test_config_file.json b/tests/test_working_directory/test_config_file.json index 79c25f0..587741c 100644 --- a/tests/test_working_directory/test_config_file.json +++ b/tests/test_working_directory/test_config_file.json @@ -1,3 +1,3 @@ -{ - "max_number_of_runs":3 +{ + "max_number_of_runs":3 } \ No newline at end of file