From 845999fdbae0699887207f8be74941b81f25408a Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 13 Jan 2025 15:17:28 -0600 Subject: [PATCH 1/6] mdformat pre-commit rules (#928) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Enable and apply mdformat pre-commit rules --- ## Type of Change - ๐Ÿ”„ Refactor --- --- .github/pull_request_template.md | 10 +- .pre-commit-config.yaml | 10 ++ CONTRIBUTING.md | 102 +++++++++--------- MAINTAINING.md | 66 ++++++------ README.md | 70 ++++++------ SECURITY.md | 16 +-- doc/README.md | 1 + mlos_bench/README.md | 81 +++++++------- .../remote/azure/scripts/setup-rg/README.md | 101 ++++++++--------- mlos_bench/mlos_bench/environments/README.md | 1 + mlos_bench/mlos_bench/services/README.md | 4 +- .../services/remote/azure/README.md | 4 +- mlos_bench/mlos_bench/storage/README.md | 1 + .../mlos_bench/storage/sql/alembic/README.md | 48 ++++----- .../tests/services/remote/ssh/README.md | 2 +- mlos_core/README.md | 20 ++-- mlos_core/mlos_core/spaces/adapters/README.md | 18 +++- 17 files changed, 293 insertions(+), 262 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c21c2a55ef7..61a485e6901 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,7 @@ _A brief, descriptive title of your changes (e.g., `Fix bug in data parser for edge cases`)_ ---- +______________________________________________________________________ ## Description @@ -12,7 +12,7 @@ _Provide a concise summary of what this PR does. Explain why this change is nece - **Issue link**: (optional) _Link the relevant issue number or GitHub Issue link if applicable (e.g., Closes #123)._ ---- +______________________________________________________________________ ## Type of Change @@ -25,16 +25,16 @@ _Indicate the type of change by choosing one (or more) of the following:_ - ๐Ÿ“ Documentation update - ๐Ÿงช Tests ---- +______________________________________________________________________ ## Testing _Describe briefly how you tested your changes._ ---- +______________________________________________________________________ ## Additional Notes (optional) _Add any additional context or information for reviewers._ ---- +______________________________________________________________________ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7311640e6e1..3a0cce4ede7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,6 +62,16 @@ repos: hooks: - id: docformatter stages: [pre-commit, manual] +- repo: https://github.com/executablebooks/mdformat + rev: 0.7.17 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-gfm + - mdformat-black + - mdformat_frontmatter + - mdformat-toc + # # Linting # diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf7bbb64992..ef9ab4fdea6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,26 +34,26 @@ Simply open the project in VSCode and follow the prompts to build and open the d 1. Create the `mlos` Conda environment. - ```sh - conda env create -f conda-envs/mlos.yml - ``` + ```sh + conda env create -f conda-envs/mlos.yml + ``` - > See the [`conda-envs/`](./conda-envs/) directory for additional conda environment files, including those used for Windows (e.g. [`mlos-windows.yml`](./conda-envs/mlos-windows.yml)). + > See the [`conda-envs/`](./conda-envs/) directory for additional conda environment files, including those used for Windows (e.g. [`mlos-windows.yml`](./conda-envs/mlos-windows.yml)). or - ```sh - # This will also ensure the environment is update to date using "conda env update -f conda-envs/mlos.yml" - make conda-env - ``` + ```sh + # This will also ensure the environment is update to date using "conda env update -f conda-envs/mlos.yml" + make conda-env + ``` - > Note: the latter expects a *nix environment. + > Note: the latter expects a \*nix environment. 1. Initialize the shell environment. - ```sh - conda activate mlos - ``` + ```sh + conda activate mlos + ``` ### Details @@ -62,45 +62,47 @@ Simply open the project in VSCode and follow the prompts to build and open the d We expect development to follow a typical "forking" style workflow: 1. Fork a copy of the [MLOS repo in Github](https://github.com/microsoft/MLOS). + 1. Create a development (a.k.a. topic) branch off of `main` to work on changes. - For instance: + For instance: - ```shell - git checkout -b YourDevName/some-topic-description main - ``` + ```shell + git checkout -b YourDevName/some-topic-description main + ``` 1. Ensure all of the lint checks and tests pass. - The easiest way to do this is to run the `make` commands that are also used in the CI pipeline: + The easiest way to do this is to run the `make` commands that are also used in the CI pipeline: - ```shell - # All at once in parallel. - make all + ```shell + # All at once in parallel. + make all - # Or individually (for easier debugging) - make format - make check - make test - make dist-test - make doc-test - ``` + # Or individually (for easier debugging) + make format + make check + make test + make dist-test + make doc-test + ``` - > Note: `make format` and `make check` use [`pre-commit`](https://pre-commit.com/) to run checks and auto-formatting. - > See the [`.pre-commit-config.yaml`](./.pre-commit-config.yaml) file for more details. - > You can also run `pre-commit install` to enable the checks in your local git hooks. - > - > See the [documentation README](./doc/README.md) for more information on documentation and its testing. + > Note: `make format` and `make check` use [`pre-commit`](https://pre-commit.com/) to run checks and auto-formatting. + > See the [`.pre-commit-config.yaml`](./.pre-commit-config.yaml) file for more details. + > You can also run `pre-commit install` to enable the checks in your local git hooks. + > + > See the [documentation README](./doc/README.md) for more information on documentation and its testing. 1. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls). - Some notes on organizing changes to help reviewers: + Some notes on organizing changes to help reviewers: - 1. Please try to keep PRs small whenver possible and don't include unnecessaary formatting changes. - 1. Larger changes can be planned in [Issues](https://github.com/microsoft/MLOS/issues), prototyped in a large draft PR for early feedback, and split into smaller PRs via discussion. - 1. All changes should include test coverage (either new or existing). + 1. Please try to keep PRs small whenver possible and don't include unnecessaary formatting changes. + 1. Larger changes can be planned in [Issues](https://github.com/microsoft/MLOS/issues), prototyped in a large draft PR for early feedback, and split into smaller PRs via discussion. + 1. All changes should include test coverage (either new or existing). 1. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass). + 1. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history. ## Distributing @@ -109,24 +111,24 @@ You can also locally build and install from wheels like so: 1. Build the *wheel* file(s) - ```sh - make dist - ``` + ```sh + make dist + ``` -2. Install it. +1. Install it. - ```sh - # this will install just the optimizer component with SMAC support: - pip install "dist/tmp/mlos_core-latest-py3-none-any.whl[smac]" - ``` + ```sh + # this will install just the optimizer component with SMAC support: + pip install "dist/tmp/mlos_core-latest-py3-none-any.whl[smac]" + ``` - ```sh - # this will install both the optimizer and the experiment runner: - pip install "dist/mlos_bench-latest-py3-none-any.whl[azure]" - ``` + ```sh + # this will install both the optimizer and the experiment runner: + pip install "dist/mlos_bench-latest-py3-none-any.whl[azure]" + ``` - > Note: exact versions may differ due to automatic versioning so the `-latest-` part is a symlink. - > If distributing elsewhere, adjust for the current version number in the module's `dist` directory. + > Note: exact versions may differ due to automatic versioning so the `-latest-` part is a symlink. + > If distributing elsewhere, adjust for the current version number in the module's `dist` directory. ### See Also diff --git a/MAINTAINING.md b/MAINTAINING.md index 9a7f6060365..1a331834ff4 100644 --- a/MAINTAINING.md +++ b/MAINTAINING.md @@ -10,51 +10,51 @@ See the [documentation README](./doc/README.md) for more information on writing 1. Bump the version using the [`update-version.sh`](./scripts/update-version.sh) script: - ```sh - git checkout -b bump-version main - ./scripts/update-version.sh --no-tag patch # or minor or major - ``` + ```sh + git checkout -b bump-version main + ./scripts/update-version.sh --no-tag patch # or minor or major + ``` - > By default this would create a local tag, but we would have to overwrite it later, so we skip that step. + > By default this would create a local tag, but we would have to overwrite it later, so we skip that step. -2. Test it! +1. Test it! - ```sh - make dist-test + ```sh + make dist-test - # Make sure that the version number on the wheels looks correct. - ls */dist/*.whl - ``` + # Make sure that the version number on the wheels looks correct. + ls */dist/*.whl + ``` -3. Make and merge a PR. +1. Make and merge a PR. -4. Update the tag locally. +1. Update the tag locally. - Once the PR with the new version files is merged. + Once the PR with the new version files is merged. - ```sh - git checkout main - git pull - git tag vM.m.p - ``` + ```sh + git checkout main + git pull + git tag vM.m.p + ``` - > Note: `M.m.p` is the version number you just bumped to above. + > Note: `M.m.p` is the version number you just bumped to above. -5. Retest! +1. Retest! - ```sh - make dist-clean - make dist-test - ``` + ```sh + make dist-clean + make dist-test + ``` -6. Update the tag remotely to the MLOS upstream repo. +1. Update the tag remotely to the MLOS upstream repo. - ```sh - git push --tags # upstream (if that's what you called your upstream git remote) - ``` + ```sh + git push --tags # upstream (if that's what you called your upstream git remote) + ``` -7. Make a "Release" on Github. +1. Make a "Release" on Github. - > Once this is done, the rules in [`.github/workflows/devcontainer.yml`](./.github/workflows/devcontainer.yml) will automatically publish the wheels to [pypi](https://pypi.org/project/mlos-core/) and tagged docker images to ACR. - > \ - > Note: This may fail if the version number is already published to pypi, in which case start from the beginning with a new patch version. + > Once this is done, the rules in [`.github/workflows/devcontainer.yml`](./.github/workflows/devcontainer.yml) will automatically publish the wheels to [pypi](https://pypi.org/project/mlos-core/) and tagged docker images to ACR. + > \ + > Note: This may fail if the version number is already published to pypi, in which case start from the beginning with a new patch version. diff --git a/README.md b/README.md index ebb717445ba..eb742bd0b4f 100644 --- a/README.md +++ b/README.md @@ -10,25 +10,24 @@ MLOS is a project to enable autotuning for systems. ## Contents - - -- [MLOS](#mlos) - - [Contents](#contents) - - [Overview](#overview) - - [Organization](#organization) - - [Contributing](#contributing) - - [Getting Started](#getting-started) - - [conda activation](#conda-activation) - - [Usage Examples](#usage-examples) - - [mlos-core](#mlos-core) - - [mlos-bench](#mlos-bench) - - [mlos-viz](#mlos-viz) - - [Installation](#installation) - - [See Also](#see-also) - - [Examples](#examples) - - [Publications](#publications) - - + + +- [Contents](#contents) +- [Overview](#overview) +- [Organization](#organization) +- [Contributing](#contributing) +- [Getting Started](#getting-started) + - [`conda` activation](#conda-activation) + - [Usage Examples](#usage-examples) + - [`mlos-core`](#mlos-core) + - [`mlos-bench`](#mlos-bench) + - [`mlos-viz`](#mlos-viz) +- [Installation](#installation) +- [See Also](#see-also) + - [Examples](#examples) + - [Publications](#publications) + + ## Overview @@ -43,9 +42,12 @@ To accomplish this, the general flow involves - Apply that configuration to the target system. - Repeat until either the exploration budget is consumed or the configurations' performance appear to have converged. - + + optimization loop + + > Source: [LlamaTune: VLDB 2022](https://arxiv.org/abs/2203.05128) For a brief overview of some of the features and capabilities of MLOS, please see the following video: @@ -60,7 +62,7 @@ To do this this repo provides three Python modules, which can be used independen - [`mlos-viz`](./mlos_viz/) provides some simple APIs to help automate visualizing the results of benchmark experiments and their trials. - It provides a simple `plot(experiment_data)` API, where `experiment_data` is obtained from the `mlos_bench.storage` module. + It provides a simple `plot(experiment_data)` API, where `experiment_data` is obtained from the `mlos_bench.storage` module. - [`mlos-core`](./mlos_core/) provides an abstraction around existing optimization frameworks (e.g., [FLAML](https://github.com/microsoft/FLAML), [SMAC](https://github.com/automl/SMAC3), etc.) @@ -92,26 +94,26 @@ For instructions on setting up the development environment please try one of the 1. Create the `mlos` Conda environment. - ```sh - conda env create -f conda-envs/mlos.yml - ``` + ```sh + conda env create -f conda-envs/mlos.yml + ``` - > See the [`conda-envs/`](./conda-envs/) directory for additional conda environment files, including those used for Windows (e.g. [`mlos-windows.yml`](./conda-envs/mlos-windows.yml)). + > See the [`conda-envs/`](./conda-envs/) directory for additional conda environment files, including those used for Windows (e.g. [`mlos-windows.yml`](./conda-envs/mlos-windows.yml)). or - ```sh - # This will also ensure the environment is update to date using "conda env update -f conda-envs/mlos.yml" - make conda-env - ``` + ```sh + # This will also ensure the environment is update to date using "conda env update -f conda-envs/mlos.yml" + make conda-env + ``` - > Note: the latter expects a *nix environment. + > Note: the latter expects a \*nix environment. 1. Initialize the shell environment. - ```sh - conda activate mlos - ``` + ```sh + conda activate mlos + ``` ### Usage Examples @@ -186,7 +188,7 @@ Details on using a local version from git are available in [CONTRIBUTING.md](./C - [sqlite-autotuning](https://github.com/Microsoft-CISL/sqlite-autotuning) - Working example of tuning `sqlite` with MLOS. + Working example of tuning `sqlite` with MLOS. These can be used as starting points for new autotuning projects outside of the main MLOS repository if you want to keep your tuning experiment configs separate from the MLOS codebase. diff --git a/SECURITY.md b/SECURITY.md index 5d6f1cfbc82..2a36bd9f82d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). -If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](), please report it to us as described below. ## Reporting Security Issues @@ -18,13 +18,13 @@ You should receive a response within 24 hours. If for some reason you do not, pl Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: - * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) - * Full paths of source file(s) related to the manifestation of the issue - * The location of the affected source code (tag/branch/commit or direct URL) - * Any special configuration required to reproduce the issue - * Step-by-step instructions to reproduce the issue - * Proof-of-concept or exploit code (if possible) - * Impact of the issue, including how an attacker might exploit the issue +- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) +- Full paths of source file(s) related to the manifestation of the issue +- The location of the affected source code (tag/branch/commit or direct URL) +- Any special configuration required to reproduce the issue +- Step-by-step instructions to reproduce the issue +- Proof-of-concept or exploit code (if possible) +- Impact of the issue, including how an attacker might exploit the issue This information will help us triage your report more quickly. diff --git a/doc/README.md b/doc/README.md index 4ed884f8cc7..78c96a647bc 100644 --- a/doc/README.md +++ b/doc/README.md @@ -63,6 +63,7 @@ class MyClass: Expected output """ + ... ``` diff --git a/mlos_bench/README.md b/mlos_bench/README.md index 0afec2ee9fb..daaf7430d49 100644 --- a/mlos_bench/README.md +++ b/mlos_bench/README.md @@ -10,59 +10,57 @@ It's available for `pip install` via the pypi repository at [mlos-bench](https:/ - - -- [mlos-bench](#mlos-bench) - - [Table of Contents](#table-of-contents) - - [Description](#description) - - [Features](#features) - - [Quickstart](#quickstart) - - [Install and activate the conda environment](#install-and-activate-the-conda-environment) - - [Make sure that you have Azure CLI tool installed and working](#make-sure-that-you-have-azure-cli-tool-installed-and-working) - - [Generate access tokens to interact with Azure resources](#generate-access-tokens-to-interact-with-azure-resources) - - [Create a JSON config with DB credentials Optional](#create-a-json-config-with-db-credentials-optional) - - [Create a top-level configuration file for your MLOS setup](#create-a-top-level-configuration-file-for-your-mlos-setup) - - [Create another config file for the parameters specific to your experiment](#create-another-config-file-for-the-parameters-specific-to-your-experiment) - - [Importance of the Experiment ID config](#importance-of-the-experiment-id-config) - - [Run the benchmark](#run-the-benchmark) - - [Optimization](#optimization) - - [Resuming interrupted experiments](#resuming-interrupted-experiments) - - [Analyzing Results](#analyzing-results) - - [Debugging](#debugging) - - [See Also](#see-also) - - - + + +- [Table of Contents](#table-of-contents) +- [Description](#description) +- [Features](#features) +- [Quickstart](#quickstart) + - [Install and activate the conda environment](#install-and-activate-the-conda-environment) + - [Make sure that you have Azure CLI tool installed and working](#make-sure-that-you-have-azure-cli-tool-installed-and-working) + - [Generate access tokens to interact with Azure resources](#generate-access-tokens-to-interact-with-azure-resources) + - [Create a JSON config with DB credentials (Optional)](#create-a-json-config-with-db-credentials-optional) + - [Create a top-level configuration file for your MLOS setup](#create-a-top-level-configuration-file-for-your-mlos-setup) + - [Create another config file for the parameters specific to your experiment](#create-another-config-file-for-the-parameters-specific-to-your-experiment) + - [Importance of the Experiment ID config](#importance-of-the-experiment-id-config) + - [Run the benchmark](#run-the-benchmark) +- [Optimization](#optimization) + - [Resuming interrupted experiments](#resuming-interrupted-experiments) +- [Analyzing Results](#analyzing-results) +- [Debugging](#debugging) +- [See Also](#see-also) + + ## Description `mlos-bench` is an end-to-end benchmarking service that can be independently launched for experimentation but is also integrated with `mlos-core` as its optimizer for OS tuning. - Given a user-provided VM configuration, `mlos-bench` provisions a configured environment and remotely executes benchmarks on the cloud. - Experiment results (benchmark results & telemetry) are stored as input to the `mlos-core` optimization engine loop to evaluate proposed configuration parameters and produce new results. +Given a user-provided VM configuration, `mlos-bench` provisions a configured environment and remotely executes benchmarks on the cloud. +Experiment results (benchmark results & telemetry) are stored as input to the `mlos-core` optimization engine loop to evaluate proposed configuration parameters and produce new results. ## Features With a [JSON5](https://spec.json5.org) [config file](./mlos_bench/config/) and command line parameters as input, `mlos-bench` streamlines workload performance measurement by automating the following benchmarking steps: 1. Set up & clean up benchmark and application configuration - - **Ease of use:** Mlos-bench abstracts away controls for managing VMs in Azure, e.g., setup, teardown, stop, deprovision, and reboot. Get visibility into VM status through Azure Portal, ensuring that a VM is provisioned & running before issuing commands to it. - - **Versatility:** Mlos-bench provides a common interface to control a collection of environments (application, OS, VM), regardless of where or which cloud they come from. This allows changes to easily propagate to all environment layers when a new set of kernel parameters are applied. - - **Efficiency:** In adapting an environment to new parameters, mlos-bench optimizes for low re-configuration costs during optimization. For example, considering that not all OS kernel parameter adjustments require a full reboot, as some can be changed during run-time. -2. Run benchmarks in the provisioned environment & standardize results for the optimizer - - Through Azure File Share, access docker scripts to run benchmarks & store results as input for optimization. For example, execute Redis benchmark uploaded to the file share, running a benchmark docker container with specified parameters. The file share is mounted to VMs via remote execution, instead of ARM templates. - - **Configurable:** Specify a python script in the initial config to post-process & standardize benchmark results. An example post-processing script for Redis benchmarks is included. - - **Local & remote benchmark execution:** Benchmarks can be run both locally in Hyper-V and remotely on Azure. Local execution allows better accuracy, while Azure runs are required to estimate the benchmark noise and understand the VM behavior when using cloud storage. - - **Cloud agnostic:** Mlos-bench can remotely execute benchmarks on other clouds, outside of Azure - e.g., controls for EC2 instances and ability to provision environments on AWS with Terraform. - - **Persistence:** Storage integration is available to persist experiment parameters and track results for re-use, either for analysis during & after trials, or warm-starting future experiments. + - **Ease of use:** Mlos-bench abstracts away controls for managing VMs in Azure, e.g., setup, teardown, stop, deprovision, and reboot. Get visibility into VM status through Azure Portal, ensuring that a VM is provisioned & running before issuing commands to it. + - **Versatility:** Mlos-bench provides a common interface to control a collection of environments (application, OS, VM), regardless of where or which cloud they come from. This allows changes to easily propagate to all environment layers when a new set of kernel parameters are applied. + - **Efficiency:** In adapting an environment to new parameters, mlos-bench optimizes for low re-configuration costs during optimization. For example, considering that not all OS kernel parameter adjustments require a full reboot, as some can be changed during run-time. +1. Run benchmarks in the provisioned environment & standardize results for the optimizer + - Through Azure File Share, access docker scripts to run benchmarks & store results as input for optimization. For example, execute Redis benchmark uploaded to the file share, running a benchmark docker container with specified parameters. The file share is mounted to VMs via remote execution, instead of ARM templates. + - **Configurable:** Specify a python script in the initial config to post-process & standardize benchmark results. An example post-processing script for Redis benchmarks is included. + - **Local & remote benchmark execution:** Benchmarks can be run both locally in Hyper-V and remotely on Azure. Local execution allows better accuracy, while Azure runs are required to estimate the benchmark noise and understand the VM behavior when using cloud storage. + - **Cloud agnostic:** Mlos-bench can remotely execute benchmarks on other clouds, outside of Azure - e.g., controls for EC2 instances and ability to provision environments on AWS with Terraform. + - **Persistence:** Storage integration is available to persist experiment parameters and track results for re-use, either for analysis during & after trials, or warm-starting future experiments. ## Quickstart To get started, we can adapt an example configuration to test out running `mlos-bench`. For these instructions, we will be using Azure for our resources. -### 1. Install and activate the conda environment +### Install and activate the conda environment From here onwards we assume we are in the project root directory. Ensure you have a conda environment (`mlos`) set up for executing `mlos_bench`. @@ -75,7 +73,7 @@ conda activate mlos > Note: if you are running inside the devcontainer, this should be done automatically. -### 2. Make sure that you have Azure CLI tool installed and working +### Make sure that you have Azure CLI tool installed and working > Installation instructions for `az` (Azure CLI) [can be found here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli). > @@ -94,7 +92,7 @@ az config set storage.account=MyStorageAccount --local az account set --subscription "..." ``` -### 3. Generate access tokens to interact with Azure resources +### Generate access tokens to interact with Azure resources A script at `./scripts/generate-azure-credentials-config` produces a JSON config snippet with necessary Azure credentials. @@ -114,7 +112,7 @@ This data produced in the `global_config_azure.jsonc` file is in the format that > Note: On Linux, this script also requires `jq` to also be installed (comes preinstalled in the devcontainer). -### 4. Create a JSON config with DB credentials (Optional) +### Create a JSON config with DB credentials (Optional) If you plan to store the information about experiments and benchmarks in a (remote) database like PostgreSQL or MySQL, create a JSON/JSONC file with the DB hostname and password. See [`mysql.jsonc`](./mlos_bench/config/storage/mysql.jsonc) or [`postgresql.jsonc`](./mlos_bench/config/storage/postgresql.jsonc) configuration files for examples with a more complete list of DB parameters supported by underlying the [SqlAlchemy](https://www.sqlalchemy.org/library.html#reference) library. @@ -136,7 +134,7 @@ See [`sqlite.jsonc`](./mlos_bench/config/storage/sqlite.jsonc) or [`duckdb.jsonc > Note: if no storage is specified, a basic sqlite config will be used by default. -### 5. Create a top-level configuration file for your MLOS setup +### Create a top-level configuration file for your MLOS setup We provide a few examples of such files in [`./mlos_bench/config/cli/`](./mlos_bench/config/cli). For example, [`azure-redis-opt.jsonc`](./mlos_bench/config/cli/azure-redis-opt.jsonc) is a configuration for optimizing Redis VM on Azure and saving the results in a local SQLite database. @@ -159,7 +157,7 @@ For the purpose of this tutorial, we will assume that we reuse the existing [`az In a more realistic scenario, however, you might need to change and/or create new config files for some parts of your benchmarking environment. We'll give more details on that below. -### 5. Create another config file for the parameters specific to your experiment +### Create another config file for the parameters specific to your experiment Copy one of our examples, e.g., [`experiment_RedisBench.jsonc`](./mlos_bench/config/experiments/experiment_RedisBench.jsonc) and name it after your experiment, e.g. `experiment_MyBenchmark.jsonc`. @@ -177,7 +175,7 @@ As such this value should be unique for each experiment. Be sure to change it whenever *"incompatible"* changes are made to the experiment configuration or scripts. Unfortunately, determining what constitutes and *"incompatible"* change for any given system is not always possible, so `mlos_bench` largely leaves this up to the user. -### 6. Run the benchmark +### Run the benchmark Now we can run our configuration with `mlos_bench`: @@ -230,6 +228,7 @@ For example: ```python from mlos_bench.storage import from_config + # Specify the experiment_id used for your experiment. experiment_id = "YourExperimentId" trial_id = 1 diff --git a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md index f3c0953d6f2..091a5d10691 100644 --- a/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md +++ b/mlos_bench/mlos_bench/config/services/remote/azure/scripts/setup-rg/README.md @@ -7,74 +7,75 @@ The *control plane RG* is a container for the *persistent* resources of MLOS (re 1. Starting in this current directory, ensure that we are logged in to Azure CLI. - ```sh - az login - ``` + ```sh + az login + ``` -2. Make a copy of the control plane ARM parameters file. +1. Make a copy of the control plane ARM parameters file. - ```sh - cp rg-template.example.parameters.json rg-template.parameters.json - ``` + ```sh + cp rg-template.example.parameters.json rg-template.parameters.json + ``` -3. (Optional) Make a copy of the results DB parameters file, if planning to provision a results DB (suggested). +1. (Optional) Make a copy of the results DB parameters file, if planning to provision a results DB (suggested). - ```sh - cp results-db/mysql-template.parameters.example.json results-db/mysql-template.parameters.json - ``` + ```sh + cp results-db/mysql-template.parameters.example.json results-db/mysql-template.parameters.json + ``` -4. Modify the ARM parameters in the newly created files as needed, especially the `PLACEHOLDER` values. +1. Modify the ARM parameters in the newly created files as needed, especially the `PLACEHOLDER` values. -5. Execute the main script with CLI args as follows: +1. Execute the main script with CLI args as follows: - ```shell - # With Powershell, recommended to use Powershell 7 - ./setup-rg.ps1 ` - -controlPlaneArmParamsFile $controlPlaneArmParamsFile ` - -resultsDbArmParamsFile $resultsDbArmParamsFile # If provisioning results DB, otherwise omit ` - -servicePrincipalName $servicePrincipalName ` - -resourceGroupName $resourceGroupName ` - -certName $certName - ``` + ```shell + # With Powershell, recommended to use Powershell 7 + ./setup-rg.ps1 ` + -controlPlaneArmParamsFile $controlPlaneArmParamsFile ` + -resultsDbArmParamsFile $resultsDbArmParamsFile # If provisioning results DB, otherwise omit ` + -servicePrincipalName $servicePrincipalName ` + -resourceGroupName $resourceGroupName ` + -certName $certName + ``` - ```sh - # With bash - # If provisioning results DB include '--resultsDbArmsParamsFile', otherwise omit - ./setup-rg.sh \ - --controlPlaneArmParamsFile $controlPlaneArmParamsFile \ - --resultsDbArmParamsFile $resultsDbArmParamsFile \ - --servicePrincipalName $servicePrincipalName \ - --resourceGroupName $resourceGroupName \ - --certName $certName - ``` + ```sh + # With bash + # If provisioning results DB include '--resultsDbArmsParamsFile', otherwise omit + ./setup-rg.sh \ + --controlPlaneArmParamsFile $controlPlaneArmParamsFile \ + --resultsDbArmParamsFile $resultsDbArmParamsFile \ + --servicePrincipalName $servicePrincipalName \ + --resourceGroupName $resourceGroupName \ + --certName $certName + ``` - where `$*ArmParamsFile` can be the corresponding `*.parameters.json` and from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). + where `$*ArmParamsFile` can be the corresponding `*.parameters.json` and from before. However, it also follows the same usage as `--parameters` in [az deployment group create](https://learn.microsoft.com/en-us/cli/azure/deployment/group?view=azure-cli-latest#az-deployment-group-create-examples). ## Workflow The high-level flow for what this script automates is as follows: 1. Assign `Contributor` access to the Service Principal (SP) for write access over resources. - Ideally, experiment resources are placed in their own RG. - When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. + Ideally, experiment resources are placed in their own RG. + When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. -2. Provision control plane resources into the RG. - This includes: - - Control VM for running the `mlos_bench` scheduler. - - Control VM's networking (public IP, security group, vnet, subnet, network interface) - - Key Vault for storing the SP credentials. - - Storage (storage account, file share) +1. Provision control plane resources into the RG. + This includes: -3. The results DB is then optionally provisioned, adding appropriate firewall rules. + - Control VM for running the `mlos_bench` scheduler. + - Control VM's networking (public IP, security group, vnet, subnet, network interface) + - Key Vault for storing the SP credentials. + - Storage (storage account, file share) -4. Assign `Key Vault Administrator` access to the current user. - This allows the current user to retrieve secrets / certificates from the VM once it is set up. - Ensure to log in as the same user in the VM. +1. The results DB is then optionally provisioned, adding appropriate firewall rules. -5. Check if the desired certificate name already exists in the key vault. +1. Assign `Key Vault Administrator` access to the current user. + This allows the current user to retrieve secrets / certificates from the VM once it is set up. + Ensure to log in as the same user in the VM. -6. If certificate does not exist yet, create or update the Service Principal (SP) with `Contributor` access for write over resources. - Ideally, experiment resources are placed in their own RG. - When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. +1. Check if the desired certificate name already exists in the key vault. -7. Otherwise, create or update the SP just with similar access as before. Now also verify that the existing certificate in the key vault matches one linked to the SP already, via thumbprint. +1. If certificate does not exist yet, create or update the Service Principal (SP) with `Contributor` access for write over resources. + Ideally, experiment resources are placed in their own RG. + When that isn't possible, they can also be placed in the control plane RG, in which case the SP can optionally be given access to the control plane RG as well. + +1. Otherwise, create or update the SP just with similar access as before. Now also verify that the existing certificate in the key vault matches one linked to the SP already, via thumbprint. diff --git a/mlos_bench/mlos_bench/environments/README.md b/mlos_bench/mlos_bench/environments/README.md index 3a3cfdb6a52..c0b5465dd0d 100644 --- a/mlos_bench/mlos_bench/environments/README.md +++ b/mlos_bench/mlos_bench/environments/README.md @@ -10,6 +10,7 @@ Each `Environment` object also keeps track of the current state of the system, a Environments can have [`Tunable`](../tunables/tunable.py) parameters and [`TunableGroups`](../tunables/tunable_groups.py) for controlling their configuration. It is generally expected that all [`Tunable`](../tunables/tunable.py) parameters within an `Environment` will have the same cost to change. + > It may therefore make sense to split a portion of the system logically across `Environments` (e.g., boot-time vs. runtime settings). Common and platform-specific functionality of the environments is implemented in [`Service`](../services/) classes. diff --git a/mlos_bench/mlos_bench/services/README.md b/mlos_bench/mlos_bench/services/README.md index 125b71be46b..55a6d056f22 100644 --- a/mlos_bench/mlos_bench/services/README.md +++ b/mlos_bench/mlos_bench/services/README.md @@ -6,8 +6,8 @@ They are (roughly) divided into two categories: - `LocalService` - A service that runs on the same machine as the scheduler component. - This may be things like executing a script for parsing the results of a benchmark run using local tools that aren't necessarily available on the target system. + This may be things like executing a script for parsing the results of a benchmark run using local tools that aren't necessarily available on the target system. - `RemoteService` - A service that runs on a remote (target) machine. - This may be things like executing a script on a remote machine to start a benchmark run. + This may be things like executing a script on a remote machine to start a benchmark run. diff --git a/mlos_bench/mlos_bench/services/remote/azure/README.md b/mlos_bench/mlos_bench/services/remote/azure/README.md index 1644f3bd952..c1b3973a85d 100644 --- a/mlos_bench/mlos_bench/services/remote/azure/README.md +++ b/mlos_bench/mlos_bench/services/remote/azure/README.md @@ -33,8 +33,8 @@ The requirements of this method are: Once the SP is setup, the flow of authentication within `mlos_bench` is as follows: 1. On initialization, retrieve the SP's certificate from the key vault as the current user and keep it in memory. -2. Log in as the SP using the certificate retrieved from the key vault. -3. During runtime when tokens are requested, we request them as the SP, reauthenticating as necessary using the certificate which is stored only in memory. +1. Log in as the SP using the certificate retrieved from the key vault. +1. During runtime when tokens are requested, we request them as the SP, reauthenticating as necessary using the certificate which is stored only in memory. In the associated JSON config, these requirements translate into providing the following parameters: diff --git a/mlos_bench/mlos_bench/storage/README.md b/mlos_bench/mlos_bench/storage/README.md index ee5876c1fd6..1e2c2467e59 100644 --- a/mlos_bench/mlos_bench/storage/README.md +++ b/mlos_bench/mlos_bench/storage/README.md @@ -26,6 +26,7 @@ For example: ```python from mlos_bench.storage import from_config + # Specify the experiment_id used for your experiment. experiment_id = "YourExperimentId" trial_id = 1 diff --git a/mlos_bench/mlos_bench/storage/sql/alembic/README.md b/mlos_bench/mlos_bench/storage/sql/alembic/README.md index f544b2b14f9..dbd7271a207 100644 --- a/mlos_bench/mlos_bench/storage/sql/alembic/README.md +++ b/mlos_bench/mlos_bench/storage/sql/alembic/README.md @@ -6,40 +6,40 @@ This document contains some notes on how to use [`alembic`](https://alembic.sqla 1. Create a blank `mlos_bench.sqlite` database file in the [`mlos_bench/storage/sql`](../) directory with the current schema using the following command: - ```sh - cd mlos_bench/storage/sql - rm mlos_bench.sqlite - mlos_bench --storage storage/sqlite.jsonc --create-update-storage-schema-only - ``` + ```sh + cd mlos_bench/storage/sql + rm mlos_bench.sqlite + mlos_bench --storage storage/sqlite.jsonc --create-update-storage-schema-only + ``` - > This allows `alembic` to automatically generate a migration script from the current schema. + > This allows `alembic` to automatically generate a migration script from the current schema. -2. Adjust the [`mlos_bench/storage/sql/schema.py`](../schema.py) file to reflect the new desired schema. +1. Adjust the [`mlos_bench/storage/sql/schema.py`](../schema.py) file to reflect the new desired schema. - > Keep each change small and atomic. - > For example, if you want to add a new column, do that in one change. - > If you want to rename a column, do that in another change. + > Keep each change small and atomic. + > For example, if you want to add a new column, do that in one change. + > If you want to rename a column, do that in another change. -3. Generate a new migration script with the following command: +1. Generate a new migration script with the following command: - ```sh - alembic revision --autogenerate -m "Descriptive text about the change." - ``` + ```sh + alembic revision --autogenerate -m "Descriptive text about the change." + ``` -4. Review the generated migration script in the [`mlos_bench/storage/sql/alembic/versions`](./versions/) directory. +1. Review the generated migration script in the [`mlos_bench/storage/sql/alembic/versions`](./versions/) directory. -5. Verify that the migration script works by running the following command: +1. Verify that the migration script works by running the following command: - ```sh - mlos_bench --storage storage/sqlite.jsonc --create-update-storage-schema-only - ``` + ```sh + mlos_bench --storage storage/sqlite.jsonc --create-update-storage-schema-only + ``` - > Normally this would be done with `alembic upgrade head`, but this command is convenient to ensure if will work with the `mlos_bench` command line interface as well. + > Normally this would be done with `alembic upgrade head`, but this command is convenient to ensure if will work with the `mlos_bench` command line interface as well. -6. If the migration script works, commit the changes to the [`mlos_bench/storage/sql/schema.py`](../schema.py) and [`mlos_bench/storage/sql/alembic/versions`](./versions/) files. +1. If the migration script works, commit the changes to the [`mlos_bench/storage/sql/schema.py`](../schema.py) and [`mlos_bench/storage/sql/alembic/versions`](./versions/) files. - > Be sure to update the latest version in the [`test_storage_schemas.py`](../../../tests/storage/test_storage_schemas.py) file as well. + > Be sure to update the latest version in the [`test_storage_schemas.py`](../../../tests/storage/test_storage_schemas.py) file as well. -7. Merge that to the `main` branch. +1. Merge that to the `main` branch. -8. Might be good to cut a new `mlos_bench` release at this point as well. +1. Might be good to cut a new `mlos_bench` release at this point as well. diff --git a/mlos_bench/mlos_bench/tests/services/remote/ssh/README.md b/mlos_bench/mlos_bench/tests/services/remote/ssh/README.md index 73026d4fc4a..4fe4216ff3e 100644 --- a/mlos_bench/mlos_bench/tests/services/remote/ssh/README.md +++ b/mlos_bench/mlos_bench/tests/services/remote/ssh/README.md @@ -7,7 +7,7 @@ To do this, we make use of the `pytest-docker` plugin to bring up the services d There are two services defined in that config: 1. `ssh-server` -2. `alt-server` +1. `alt-server` We rely on `docker compose` to map their internal container service ports to random ports on the host. Hence, when connecting, we need to look up these ports on demand using something akin to `docker compose port`. diff --git a/mlos_core/README.md b/mlos_core/README.md index a33b861d480..c98689888fd 100644 --- a/mlos_core/README.md +++ b/mlos_core/README.md @@ -15,16 +15,16 @@ These can be evaluated by [`mlos-bench`](../mlos_bench/), generating and trackin Since the tunable parameter search space is often extremely large, `mlos-core` automates the following steps to efficiently generate optimal task-specific kernel and application configurations. 1. Reduce the search space by identifying a promising set of tunable parameters - - Map out the configuration search space: Automatically track and manage the discovery of new Linux kernel parameters and their default values across versions. - Filter out non-tunable parameters (e.g., not writable) and track which kernel parameters exist for a given kernel version. - - Leverage parameter knowledge for optimization: Information on ranges, sampling intervals, parameter correlations, workload type sensitivities for tunable parameters are tracked and currently manually curated. - In the future, this can be automatically maintained by scraping documentation pages on kernel parameters. - - Tailored to application: Consider prior knowledge of the parameter's impact & an application's workload profile (e.g. network heavy, disk heavy, CPU bound, multi-threaded, latency sensitive, throughput oriented, etc.) to identify likely impactful candidates of tunable parameters, specific to a particular application. -2. Sampling to warm-start optimization in a high dimensional search space -3. Produce optimal configurations through Bayesian optimization - - Support for various optimizer algorithms (default Bayesian optimizer, Flaml, SMAC, and random for baseline comparison), that handle multiple types of constraints. - This includes cost-aware optimization, that considers experiment costs given current tunable parameters. - - Integrated with `mlos-bench`, proposed configurations are logged and evaluated. + - Map out the configuration search space: Automatically track and manage the discovery of new Linux kernel parameters and their default values across versions. + Filter out non-tunable parameters (e.g., not writable) and track which kernel parameters exist for a given kernel version. + - Leverage parameter knowledge for optimization: Information on ranges, sampling intervals, parameter correlations, workload type sensitivities for tunable parameters are tracked and currently manually curated. + In the future, this can be automatically maintained by scraping documentation pages on kernel parameters. + - Tailored to application: Consider prior knowledge of the parameter's impact & an application's workload profile (e.g. network heavy, disk heavy, CPU bound, multi-threaded, latency sensitive, throughput oriented, etc.) to identify likely impactful candidates of tunable parameters, specific to a particular application. +1. Sampling to warm-start optimization in a high dimensional search space +1. Produce optimal configurations through Bayesian optimization + - Support for various optimizer algorithms (default Bayesian optimizer, Flaml, SMAC, and random for baseline comparison), that handle multiple types of constraints. + This includes cost-aware optimization, that considers experiment costs given current tunable parameters. + - Integrated with `mlos-bench`, proposed configurations are logged and evaluated. ## See Also diff --git a/mlos_core/mlos_core/spaces/adapters/README.md b/mlos_core/mlos_core/spaces/adapters/README.md index 6d1ac979783..3b3dc6f513f 100644 --- a/mlos_core/mlos_core/spaces/adapters/README.md +++ b/mlos_core/mlos_core/spaces/adapters/README.md @@ -38,9 +38,23 @@ The user can employ `LlamaTuneSpaceAdapter` when initializing the `mlos_core` op ```python llamatune_optimizer = OptimizerFactory.create( - ... + parameter_space=ConfigurationSpace(), + optimizer_type=OptimizerType.SMAC, + optimization_targets=["score"], + optimizer_kwargs={ + # optimizer kwargs + # e.g., + max_trials=100, + }, space_adapter_type=SpaceAdapterType.LLAMATUNE, - space_adapter_kwargs=, + space_adapter_kwargs={ + # llamatune kwargs + # e.g., + num_low_dims=16, + special_param_values={ + "special_param": [-1, 0], + }, + }, ) ``` From 3c319cd97004a098dfd2e9074922e3242d8022b5 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 13 Jan 2025 15:39:39 -0600 Subject: [PATCH 2/6] Enable pre-commit autoupdate PRs (#929) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Enable pre-commit autoupdate PRs --- ## Description Adds configs for pre-commit.ci to use to scan our repo and automatically submit PRs to update pre-commit hook version monthly. --- ## Type of Change - โœจ New feature --- ## Additional Notes (optional) This also enables autofix PRs to hopefully improve the turnaround time on PR reviews by automatically applying formatting fixups to a PR. This could get confusing or annoying if PR authors don't locally pull after fixes are applied or if the formatters do something drastic that we aren't anticipating, so we may consider to disable this in the future as well. --- --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sergiy Matusevych --- .pre-commit-config.yaml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a0cce4ede7..6553c287a35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,23 @@ +# Enable automatic updates via Github Actions as well. +# See https://pre-commit.ci +ci: + # Let pre-commit.ci automatically update PRs with formatting fixes. + autofix_prs: true + # skip local hooks - they should be managed manually via conda-envs/*.yml + skip: [mypy, pylint, pycodestyle] + autoupdate_schedule: monthly + autoupdate_commit_msg: | + [pre-commit.ci] pre-commit autoupdate + + for more information, see https://pre-commit.ci + + NOTE: Be sure to also check for other possible hook updates in the conda-envs/*.yml files (e.g., mypy, pylint, etc.). + See Also: + - https://github.com/microsoft/MLOS/blob/main/conda-envs/mlos.yml + - https://pypi.org/project/mypy/ + - https://pypi.org/project/pylint/ + - https://pypi.org/project/pycodestyle/ + # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks default_stages: [pre-commit] @@ -6,7 +26,7 @@ default_stages: [pre-commit] # are partitioned and the hook executable called in parallel across them, not # whether hooks themselves are parallelized. # As such, some hooks (e.g., pylint) which do internal parallelism need it set -# for effeciency and correctness anyways. +# for efficiency and correctness anyways. repos: # From 0cab884c09399f478418ef0dbdd7bc70fe5c4b7f Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 16 Jan 2025 18:13:42 -0600 Subject: [PATCH 3/6] Schema changes example for mlos_benchd service (#931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Schema changes for mlos_benchd service. ______________________________________________________________________ ## Description Schema changes for mlos_benchd service. Storage APIs to adjust these to come in a future PR. - See #732 ______________________________________________________________________ ## Type of Change - โœจ New feature ______________________________________________________________________ ## Testing Local, CI ______________________________________________________________________ ## Additional Notes (optional) @eujing have a look at the [commit history in the PR](https://github.com/microsoft/MLOS/pull/931/commits) for a sense of what's going on. This can probably be merged as is. Happy to discuss further though. ______________________________________________________________________ --- mlos_bench/mlos_bench/environments/status.py | 32 +++++++---- .../mlos_bench/storage/sql/alembic/README.md | 7 +++ ...15b_adding_experiment_table_columns_to_.py | 56 +++++++++++++++++++ mlos_bench/mlos_bench/storage/sql/schema.py | 13 +++++ .../tests/storage/test_storage_schemas.py | 2 +- 5 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 mlos_bench/mlos_bench/storage/sql/alembic/versions/8928a401115b_adding_experiment_table_columns_to_.py diff --git a/mlos_bench/mlos_bench/environments/status.py b/mlos_bench/mlos_bench/environments/status.py index f3e0d0ea377..ca35b3473da 100644 --- a/mlos_bench/mlos_bench/environments/status.py +++ b/mlos_bench/mlos_bench/environments/status.py @@ -2,13 +2,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Enum for the status of the benchmark/environment.""" +"""Enum for the status of the benchmark/environment Trial or Experiment.""" import enum class Status(enum.Enum): - """Enum for the status of the benchmark/environment.""" + """Enum for the status of the benchmark/environment Trial or Experiment.""" UNKNOWN = 0 PENDING = 1 @@ -29,8 +29,8 @@ def is_good(self) -> bool: } def is_completed(self) -> bool: - """Check if the status of the benchmark/environment is one of {SUCCEEDED, - CANCELED, FAILED, TIMED_OUT}. + """Check if the status of the benchmark/environment Trial or Experiment is one + of {SUCCEEDED, CANCELED, FAILED, TIMED_OUT}. """ return self in { Status.SUCCEEDED, @@ -40,25 +40,37 @@ def is_completed(self) -> bool: } def is_pending(self) -> bool: - """Check if the status of the benchmark/environment is PENDING.""" + """Check if the status of the benchmark/environment Trial or Experiment is + PENDING. + """ return self == Status.PENDING def is_ready(self) -> bool: - """Check if the status of the benchmark/environment is READY.""" + """Check if the status of the benchmark/environment Trial or Experiment is + READY. + """ return self == Status.READY def is_succeeded(self) -> bool: - """Check if the status of the benchmark/environment is SUCCEEDED.""" + """Check if the status of the benchmark/environment Trial or Experiment is + SUCCEEDED. + """ return self == Status.SUCCEEDED def is_failed(self) -> bool: - """Check if the status of the benchmark/environment is FAILED.""" + """Check if the status of the benchmark/environment Trial or Experiment is + FAILED. + """ return self == Status.FAILED def is_canceled(self) -> bool: - """Check if the status of the benchmark/environment is CANCELED.""" + """Check if the status of the benchmark/environment Trial or Experiment is + CANCELED. + """ return self == Status.CANCELED def is_timed_out(self) -> bool: - """Check if the status of the benchmark/environment is TIMED_OUT.""" + """Check if the status of the benchmark/environment Trial or Experiment is + TIMED_OUT. + """ return self == Status.FAILED diff --git a/mlos_bench/mlos_bench/storage/sql/alembic/README.md b/mlos_bench/mlos_bench/storage/sql/alembic/README.md index dbd7271a207..ec35eb70f64 100644 --- a/mlos_bench/mlos_bench/storage/sql/alembic/README.md +++ b/mlos_bench/mlos_bench/storage/sql/alembic/README.md @@ -36,6 +36,13 @@ This document contains some notes on how to use [`alembic`](https://alembic.sqla > Normally this would be done with `alembic upgrade head`, but this command is convenient to ensure if will work with the `mlos_bench` command line interface as well. + Examine the results using something like: + + ```sh + sqlite3 mlos_bench.sqlite .schema + sqlite3 mlos_bench.sqlite "SELECT * FROM alembic_version;" + ``` + 1. If the migration script works, commit the changes to the [`mlos_bench/storage/sql/schema.py`](../schema.py) and [`mlos_bench/storage/sql/alembic/versions`](./versions/) files. > Be sure to update the latest version in the [`test_storage_schemas.py`](../../../tests/storage/test_storage_schemas.py) file as well. diff --git a/mlos_bench/mlos_bench/storage/sql/alembic/versions/8928a401115b_adding_experiment_table_columns_to_.py b/mlos_bench/mlos_bench/storage/sql/alembic/versions/8928a401115b_adding_experiment_table_columns_to_.py new file mode 100644 index 00000000000..40e8f97d8e9 --- /dev/null +++ b/mlos_bench/mlos_bench/storage/sql/alembic/versions/8928a401115b_adding_experiment_table_columns_to_.py @@ -0,0 +1,56 @@ +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +"""Adding Experiment table columns to support mlos_benchd service - See #732 + +Revision ID: 8928a401115b +Revises: f83fb8ae7fc4 +Create Date: 2025-01-14 17:06:36.181503+00:00 + +""" +# pylint: disable=no-member + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "8928a401115b" +down_revision: str | None = "f83fb8ae7fc4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """The schema upgrade script for this revision.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("experiment", sa.Column("ts_start", sa.DateTime(), nullable=True)) + op.add_column("experiment", sa.Column("ts_end", sa.DateTime(), nullable=True)) + op.add_column("experiment", sa.Column("status", sa.String(length=16), nullable=True)) + op.add_column( + "experiment", + sa.Column( + "driver_name", + sa.String(length=40), + nullable=True, + comment="Driver Host/Container Name", + ), + ) + op.add_column( + "experiment", + sa.Column("driver_pid", sa.Integer(), nullable=True, comment="Driver Process ID"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """The schema downgrade script for this revision.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("experiment", "driver_pid") + op.drop_column("experiment", "driver_name") + op.drop_column("experiment", "status") + op.drop_column("experiment", "ts_end") + op.drop_column("experiment", "ts_start") + # ### end Alembic commands ### diff --git a/mlos_bench/mlos_bench/storage/sql/schema.py b/mlos_bench/mlos_bench/storage/sql/schema.py index e79c602131a..2bc00f00825 100644 --- a/mlos_bench/mlos_bench/storage/sql/schema.py +++ b/mlos_bench/mlos_bench/storage/sql/schema.py @@ -103,6 +103,19 @@ def __init__(self, engine: Engine | None): Column("root_env_config", String(1024), nullable=False), Column("git_repo", String(1024), nullable=False), Column("git_commit", String(40), nullable=False), + # For backwards compatibility, we allow NULL for ts_start. + Column("ts_start", DateTime), + Column("ts_end", DateTime), + # Should match the text IDs of `mlos_bench.environments.Status` enum: + # For backwards compatibility, we allow NULL for status. + Column("status", String(self._STATUS_LEN)), + # There may be more than one mlos_benchd_service running on different hosts. + # This column stores the host/container name of the driver that + # picked up the experiment. + # They should use a transaction to update it to their own hostname when + # they start if and only if its NULL. + Column("driver_name", String(40), comment="Driver Host/Container Name"), + Column("driver_pid", Integer, comment="Driver Process ID"), PrimaryKeyConstraint("exp_id"), ) """The Table storing diff --git a/mlos_bench/mlos_bench/tests/storage/test_storage_schemas.py b/mlos_bench/mlos_bench/tests/storage/test_storage_schemas.py index e11ff8613d9..8a6c36e6bb3 100644 --- a/mlos_bench/mlos_bench/tests/storage/test_storage_schemas.py +++ b/mlos_bench/mlos_bench/tests/storage/test_storage_schemas.py @@ -12,7 +12,7 @@ # NOTE: This value is hardcoded to the latest revision in the alembic versions directory. # It could also be obtained programmatically using the "alembic heads" command or heads() API. # See Also: schema.py for an example of programmatic alembic config access. -CURRENT_ALEMBIC_HEAD = "f83fb8ae7fc4" +CURRENT_ALEMBIC_HEAD = "8928a401115b" def test_storage_schemas(storage: SqlStorage) -> None: From 662b79d75ea54e8062d1fb7b621419c4b9d82c23 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 17 Jan 2025 14:08:03 -0600 Subject: [PATCH 4/6] Fixups for running pytest via VSCode (#933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Fixups for running pytest via VSCode ______________________________________________________________________ ## Description @motus noticed that running all tests in VSCode was hanging inside a DevContainer on MacOS. After some debugging I determined that there was some mismatch between discovering tests and running them, specifically for the GridSearchOptimizer. This was true on the CLI as well. Adding it to the `__init__.py` for that submodule seems to fix the issue. ______________________________________________________________________ ## Type of Change - ๐Ÿ› ๏ธ Bug fix ______________________________________________________________________ ## Testing - Local VSCode in both DevContainer and MacOS native mode. ______________________________________________________________________ ## Additional Notes (optional) Don't know of others like this but I wonder if that's an issue now ... ______________________________________________________________________ --- mlos_bench/mlos_bench/optimizers/__init__.py | 6 ++++-- .../test-cases/good/partial/grid_search_opt_minimal.jsonc | 2 +- .../config/schemas/optimizers/test_optimizer_schemas.py | 4 ++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mlos_bench/mlos_bench/optimizers/__init__.py b/mlos_bench/mlos_bench/optimizers/__init__.py index 7f2d6310c9a..faf186447f5 100644 --- a/mlos_bench/mlos_bench/optimizers/__init__.py +++ b/mlos_bench/mlos_bench/optimizers/__init__.py @@ -10,15 +10,17 @@ """ from mlos_bench.optimizers.base_optimizer import Optimizer +from mlos_bench.optimizers.grid_search_optimizer import GridSearchOptimizer from mlos_bench.optimizers.manual_optimizer import ManualOptimizer from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer from mlos_bench.optimizers.mock_optimizer import MockOptimizer from mlos_bench.optimizers.one_shot_optimizer import OneShotOptimizer __all__ = [ - "Optimizer", + "GridSearchOptimizer", "ManualOptimizer", + "MlosCoreOptimizer", "MockOptimizer", "OneShotOptimizer", - "MlosCoreOptimizer", + "Optimizer", ] diff --git a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc index 297e3a62c8e..0311708350f 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/partial/grid_search_opt_minimal.jsonc @@ -1,4 +1,4 @@ { - "class": "mlos_bench.optimizers.grid_search_optimizer.GridSearchOptimizer" + "class": "mlos_bench.optimizers.GridSearchOptimizer" // no config required } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test_optimizer_schemas.py b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test_optimizer_schemas.py index 9cbad7b2732..487a269e25b 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test_optimizer_schemas.py +++ b/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test_optimizer_schemas.py @@ -151,3 +151,7 @@ def test_optimizer_configs_with_extra_param(test_case_name: str) -> None: TEST_CASES.by_type["good"][test_case_name], ConfigSchema.UNIFIED, ) + + +if __name__ == "__main__": + pytest.main(args=["-n0", "-k", "grid_search_optimizer"]) From c66f793115690e37de21f6d290675dbf6d5a1d73 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 17 Jan 2025 14:31:09 -0600 Subject: [PATCH 5/6] Json Checking (#930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Json Checking ______________________________________________________________________ ## Description Add pre-commit hooks to check ~and format~ json/json5. - [x] remove prettier config - [x] check json5 files - [ ] ~preserve comments when formatting~ - [ ] ~rename mlos_bench config files to `*.mlos.jsonc`~ ______________________________________________________________________ ## Type of Change - โœจ New feature ______________________________________________________________________ ## Additional Comments See #882 for comments regarding the difficulty in auto-formatting jsonc/json5 while also preserving comments. We leave this for future work. ______________________________________________________________________ --- .pre-commit-config.yaml | 22 ++++++++++++++++++---- .prettierrc | 4 ---- 2 files changed, 18 insertions(+), 8 deletions(-) delete mode 100644 .prettierrc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6553c287a35..3bbb03cc335 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,12 +46,26 @@ repos: - id: check-yaml - id: end-of-file-fixer stages: [pre-commit, manual] - # TODO: - #- id: pretty-format-json - # args: [--autofix, --no-sort-keys] - # stages: [pre-commit, manual] - id: trailing-whitespace stages: [pre-commit, manual] +- repo: https://gitlab.com/bmares/check-json5 + rev: v1.0.0 + hooks: + - id: check-json5 + alias: check-json + types_or: [file, json] + files: '\.json[c]?$' +# FIXME: This removes comments. +# See Also: https://github.com/dpranke/pyjson5/issues/28 +#- repo: https://github.com/whtsky/pre-commit-pretty-format-json5 +# rev: "1.0.0" +# hooks: +# - id: pretty-format-json5 +# alias: pretty-format-json +# args: [--no-sort-keys] +# stages: [pre-commit, manual] +# types_or: [file, json] +# files: '\.json[c]?$' - repo: https://github.com/johann-petrak/licenseheaders rev: v0.8.8 hooks: diff --git a/.prettierrc b/.prettierrc deleted file mode 100644 index cf0454734c8..00000000000 --- a/.prettierrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "editorconfig": true, - "trailingComma": "es5" -} From e91546e6c80f966014367c3f3ecc2ec246ca8051 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 17 Jan 2025 15:31:10 -0600 Subject: [PATCH 6/6] Pyright Improvements (#932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Pyright improvements ______________________________________________________________________ ## Description Pyright is a type checker that ships with VSCode's Pylance by default. It is billed as a faster, though less complete, version of mypy. As such it gets a few things a little differently that mypy and alerts in VSCode. This PR fixes those ("standard") alerts and removes the mypy extension from VSCode's default extensions for MLOS in favor of just using pyright (there's no sense in running both interactively). We do not enable pyright's "strict" mode. Additionally, it enables pyright in pre-commit rules to ensure those fixes remain. We leave the rest of the mypy checks as well since they are still useful. A list of some of the types of fixes: - TypeDict initialization checks for Tunables - Check that json.loads() returns a dict and not a list (e.g.) - Replace ConcreteOptimizer TypeVar with a TypeAlias - Add BoundMethod protocol for checking __self__ attribute - Ensure correct type inference in a number of places - Add `...` to Protocol methods to make pyright aware of the lack of method body. - Fix a few type annotations ______________________________________________________________________ ## Type of Change - ๐Ÿ› ๏ธ Bug fix - ๐Ÿ”„ Refactor ______________________________________________________________________ ## Testing - Additional CI checks as described above. ______________________________________________________________________ --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 1 - .pre-commit-config.yaml | 17 ++- .vscode/extensions.json | 1 - .vscode/settings.json | 3 +- conda-envs/mlos.yml | 1 + doc/source/conf.py | 4 + .../mlos_bench/environments/mock_env.py | 6 +- .../mlos_bench/optimizers/mock_optimizer.py | 2 +- .../mlos_bench/services/base_service.py | 3 +- .../services/remote/ssh/ssh_fileshare.py | 2 + .../services/remote/ssh/ssh_host_service.py | 1 + .../services/types/authenticator_type.py | 6 + .../mlos_bench/services/types/bound_method.py | 24 ++++ .../services/types/config_loader_type.py | 10 ++ .../services/types/host_ops_type.py | 6 + .../services/types/host_provisioner_type.py | 6 + .../services/types/local_exec_type.py | 7 +- .../types/network_provisioner_type.py | 5 + .../mlos_bench/services/types/os_ops_type.py | 5 + .../services/types/remote_config_type.py | 4 + .../services/types/remote_exec_type.py | 4 + .../services/types/vm_provisioner_type.py | 9 ++ .../mlos_bench/storage/sql/experiment.py | 6 +- .../composite_env_service_test.py | 2 + .../tests/event_loop_context_test.py | 10 +- .../mlos_bench/tests/launcher_run_test.py | 28 +++-- .../mlos_bench/tests/services/mock_service.py | 4 + .../services/remote/mock/mock_vm_service.py | 2 + .../tests/tunables/tunable_definition_test.py | 21 ++++ .../tunables/tunable_distributions_test.py | 1 + .../tunables/tunable_slice_references_test.py | 3 + .../tests/tunables/tunables_assign_test.py | 1 + .../tests/tunables/tunables_copy_test.py | 2 + mlos_bench/mlos_bench/tunables/tunable.py | 116 ++++++++++++++---- mlos_bench/setup.py | 3 +- mlos_core/mlos_core/__init__.py | 2 +- mlos_core/mlos_core/optimizers/__init__.py | 17 +-- .../bayesian_optimizers/smac_optimizer.py | 8 +- .../mlos_core/spaces/adapters/__init__.py | 20 +-- .../mlos_core/spaces/adapters/llamatune.py | 5 +- .../mlos_core/spaces/converters/flaml.py | 5 +- mlos_core/mlos_core/spaces/converters/util.py | 11 +- mlos_core/mlos_core/tests/__init__.py | 5 +- .../tests/optimizers/optimizer_test.py | 3 +- .../adapters/space_adapter_factory_test.py | 9 +- .../mlos_core/tests/spaces/spaces_test.py | 4 +- mlos_core/setup.py | 3 +- mlos_viz/mlos_viz/base.py | 2 +- mlos_viz/setup.py | 3 +- pyproject.toml | 6 + 50 files changed, 323 insertions(+), 106 deletions(-) create mode 100644 mlos_bench/mlos_bench/services/types/bound_method.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 91373d000a5..f9cbbc4a384 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -68,7 +68,6 @@ "huntertran.auto-markdown-toc", "ibm.output-colorizer", "lextudio.restructuredtext", - "matangover.mypy", "ms-azuretools.vscode-docker", "ms-python.black-formatter", "ms-python.pylint", diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bbb03cc335..8e947e111bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ ci: # Let pre-commit.ci automatically update PRs with formatting fixes. autofix_prs: true # skip local hooks - they should be managed manually via conda-envs/*.yml - skip: [mypy, pylint, pycodestyle] + skip: [mypy, pylint, pycodestyle, pyright] autoupdate_schedule: monthly autoupdate_commit_msg: | [pre-commit.ci] pre-commit autoupdate @@ -15,6 +15,7 @@ ci: See Also: - https://github.com/microsoft/MLOS/blob/main/conda-envs/mlos.yml - https://pypi.org/project/mypy/ + - https://pypi.org/project/pyright/ - https://pypi.org/project/pylint/ - https://pypi.org/project/pycodestyle/ @@ -140,6 +141,20 @@ repos: (?x)^( doc/source/conf.py )$ + - id: pyright + name: pyright + entry: pyright + language: system + types: [python] + require_serial: true + exclude: | + (?x)^( + doc/source/conf.py| + mlos_core/setup.py| + mlos_bench/setup.py| + mlos_viz/setup.py| + conftest.py + )$ - id: mypy name: mypy entry: mypy diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 92725b501c0..ed56e7c520d 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -12,7 +12,6 @@ "huntertran.auto-markdown-toc", "ibm.output-colorizer", "lextudio.restructuredtext", - "matangover.mypy", "ms-azuretools.vscode-docker", "ms-python.black-formatter", "ms-python.pylint", diff --git a/.vscode/settings.json b/.vscode/settings.json index 406600fa4aa..23e43fed683 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -170,6 +170,5 @@ "python.testing.unittestEnabled": false, "debugpy.debugJustMyCode": false, "python.analysis.autoImportCompletions": true, - "python.analysis.supportRestructuredText": true, - "python.analysis.typeCheckingMode": "standard" + "python.analysis.supportRestructuredText": true } diff --git a/conda-envs/mlos.yml b/conda-envs/mlos.yml index 4d8f82390c9..96834c879d0 100644 --- a/conda-envs/mlos.yml +++ b/conda-envs/mlos.yml @@ -28,6 +28,7 @@ dependencies: - pylint==3.3.3 - tomlkit - mypy==1.14.1 + - pyright==1.1.392.post0 - pandas-stubs - types-beautifulsoup4 - types-colorama diff --git a/doc/source/conf.py b/doc/source/conf.py index 5ed19fc45e6..5ab3ce8bd0d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -29,6 +29,10 @@ from sphinx.application import Sphinx as SphinxApp from sphinx.environment import BuildEnvironment +# Note: doc requirements aren't installed by default. +# To install them, run `pip install -r doc/requirements.txt` + + sys.path.insert(0, os.path.abspath("../../mlos_core/mlos_core")) sys.path.insert(1, os.path.abspath("../../mlos_bench/mlos_bench")) sys.path.insert(1, os.path.abspath("../../mlos_viz/mlos_viz")) diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py index 23ec37a6392..4ddd9dee1c0 100644 --- a/mlos_bench/mlos_bench/environments/mock_env.py +++ b/mlos_bench/mlos_bench/environments/mock_env.py @@ -64,8 +64,8 @@ def __init__( # pylint: disable=too-many-arguments seed = int(self.config.get("mock_env_seed", -1)) self._run_random = random.Random(seed or None) if seed >= 0 else None self._status_random = random.Random(seed or None) if seed >= 0 else None - self._range = self.config.get("mock_env_range") - self._metrics = self.config.get("mock_env_metrics", ["score"]) + self._range: tuple[int, int] | None = self.config.get("mock_env_range") + self._metrics: list[str] | None = self.config.get("mock_env_metrics", ["score"]) self._is_ready = True def _produce_metrics(self, rand: random.Random | None) -> dict[str, TunableValue]: @@ -80,7 +80,7 @@ def _produce_metrics(self, rand: random.Random | None) -> dict[str, TunableValue if self._range: score = self._range[0] + score * (self._range[1] - self._range[0]) - return {metric: score for metric in self._metrics} + return {metric: float(score) for metric in self._metrics or []} def run(self) -> tuple[Status, datetime, dict[str, TunableValue] | None]: """ diff --git a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py index 69176ebc8d8..1e138284b1e 100644 --- a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py @@ -32,7 +32,7 @@ def __init__( self._random: dict[str, Callable[[Tunable], TunableValue]] = { "categorical": lambda tunable: rnd.choice(tunable.categories), "float": lambda tunable: rnd.uniform(*tunable.range), - "int": lambda tunable: rnd.randint(*tunable.range), + "int": lambda tunable: rnd.randint(*(int(x) for x in tunable.range)), } def bulk_register( diff --git a/mlos_bench/mlos_bench/services/base_service.py b/mlos_bench/mlos_bench/services/base_service.py index 24e9d493078..41eebfbb98b 100644 --- a/mlos_bench/mlos_bench/services/base_service.py +++ b/mlos_bench/mlos_bench/services/base_service.py @@ -14,6 +14,7 @@ from typing import Any, Literal from mlos_bench.config.schemas import ConfigSchema +from mlos_bench.services.types.bound_method import BoundMethod from mlos_bench.services.types.config_loader_type import SupportsConfigLoading from mlos_bench.util import instantiate_from_config @@ -278,7 +279,7 @@ def register(self, services: dict[str, Callable] | list[Callable]) -> None: for _, svc_method in self._service_methods.items() # Note: some methods are actually stand alone functions, so we need # to filter them out. - if hasattr(svc_method, "__self__") and isinstance(svc_method.__self__, Service) + if isinstance(svc_method, BoundMethod) and isinstance(svc_method.__self__, Service) } def export(self) -> dict[str, Callable]: diff --git a/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py b/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py index b03006e431c..8d9d35883af 100644 --- a/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py +++ b/mlos_bench/mlos_bench/services/remote/ssh/ssh_fileshare.py @@ -26,6 +26,8 @@ class CopyMode(Enum): class SshFileShareService(FileShareService, SshService): """A collection of functions for interacting with SSH servers as file shares.""" + # pylint: disable=too-many-ancestors + async def _start_file_copy( self, params: dict, diff --git a/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py b/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py index 46cae33d2b9..1b0c7e38231 100644 --- a/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py +++ b/mlos_bench/mlos_bench/services/remote/ssh/ssh_host_service.py @@ -24,6 +24,7 @@ class SshHostService(SshService, SupportsOSOps, SupportsRemoteExec): """Helper methods to manage machines via SSH.""" + # pylint: disable=too-many-ancestors # pylint: disable=too-many-instance-attributes def __init__( diff --git a/mlos_bench/mlos_bench/services/types/authenticator_type.py b/mlos_bench/mlos_bench/services/types/authenticator_type.py index 45b056119f0..e3f2135d146 100644 --- a/mlos_bench/mlos_bench/services/types/authenticator_type.py +++ b/mlos_bench/mlos_bench/services/types/authenticator_type.py @@ -14,6 +14,9 @@ class SupportsAuth(Protocol[T_co]): """Protocol interface for authentication for the cloud services.""" + # Needed by pyright + # pylint: disable=unnecessary-ellipsis,redundant-returns-doc + def get_access_token(self) -> str: """ Get the access token for cloud services. @@ -23,6 +26,7 @@ def get_access_token(self) -> str: access_token : str Access token. """ + ... def get_auth_headers(self) -> dict: """ @@ -33,6 +37,7 @@ def get_auth_headers(self) -> dict: access_header : dict HTTP header containing the access token. """ + ... def get_credential(self) -> T_co: """ @@ -43,3 +48,4 @@ def get_credential(self) -> T_co: credential : T_co Cloud-specific credential object. """ + ... diff --git a/mlos_bench/mlos_bench/services/types/bound_method.py b/mlos_bench/mlos_bench/services/types/bound_method.py new file mode 100644 index 00000000000..8e6179cffe3 --- /dev/null +++ b/mlos_bench/mlos_bench/services/types/bound_method.py @@ -0,0 +1,24 @@ +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +"""Protocol representing a bound method.""" + +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class BoundMethod(Protocol): + """A callable method bound to an object.""" + + # pylint: disable=too-few-public-methods + # pylint: disable=unnecessary-ellipsis + + @property + def __self__(self) -> Any: + """The self object of the bound method.""" + ... + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + """Call the bound method.""" + ... diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py index 71b68947f9e..97f7c98e216 100644 --- a/mlos_bench/mlos_bench/services/types/config_loader_type.py +++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py @@ -23,6 +23,9 @@ class SupportsConfigLoading(Protocol): """Protocol interface for helper functions to lookup and load configs.""" + # Needed by pyright + # pylint: disable=unnecessary-ellipsis,redundant-returns-doc + def get_config_paths(self) -> list[str]: """ Gets the list of config paths this service will search for config files. @@ -31,6 +34,7 @@ def get_config_paths(self) -> list[str]: ------- list[str] """ + ... def resolve_path(self, file_path: str, extra_paths: Iterable[str] | None = None) -> str: """ @@ -49,6 +53,7 @@ def resolve_path(self, file_path: str, extra_paths: Iterable[str] | None = None) path : str An actual path to the config or script. """ + ... def load_config( self, @@ -71,6 +76,7 @@ def load_config( config : Union[dict, list[dict]] Free-format dictionary that contains the configuration. """ + ... def build_environment( # pylint: disable=too-many-arguments self, @@ -108,6 +114,7 @@ def build_environment( # pylint: disable=too-many-arguments env : Environment An instance of the `Environment` class initialized with `config`. """ + ... def load_environment( self, @@ -140,6 +147,7 @@ def load_environment( env : Environment A new benchmarking environment. """ + ... def load_environment_list( self, @@ -173,6 +181,7 @@ def load_environment_list( env : list[Environment] A list of new benchmarking environments. """ + ... def load_services( self, @@ -198,3 +207,4 @@ def load_services( service : Service A collection of service methods. """ + ... diff --git a/mlos_bench/mlos_bench/services/types/host_ops_type.py b/mlos_bench/mlos_bench/services/types/host_ops_type.py index 4f649166fc6..29e1cd0f941 100644 --- a/mlos_bench/mlos_bench/services/types/host_ops_type.py +++ b/mlos_bench/mlos_bench/services/types/host_ops_type.py @@ -14,6 +14,8 @@ class SupportsHostOps(Protocol): """Protocol interface for Host/VM boot operations.""" + # pylint: disable=unnecessary-ellipsis + def start_host(self, params: dict) -> tuple["Status", dict]: """ Start a Host/VM. @@ -29,6 +31,7 @@ def start_host(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def stop_host(self, params: dict, force: bool = False) -> tuple["Status", dict]: """ @@ -47,6 +50,7 @@ def stop_host(self, params: dict, force: bool = False) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def restart_host(self, params: dict, force: bool = False) -> tuple["Status", dict]: """ @@ -65,6 +69,7 @@ def restart_host(self, params: dict, force: bool = False) -> tuple["Status", dic A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def wait_host_operation(self, params: dict) -> tuple["Status", dict]: """ @@ -85,3 +90,4 @@ def wait_host_operation(self, params: dict) -> tuple["Status", dict]: Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} Result is info on the operation runtime if SUCCEEDED, otherwise {}. """ + ... diff --git a/mlos_bench/mlos_bench/services/types/host_provisioner_type.py b/mlos_bench/mlos_bench/services/types/host_provisioner_type.py index 2a343877abc..d979029def6 100644 --- a/mlos_bench/mlos_bench/services/types/host_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/host_provisioner_type.py @@ -14,6 +14,8 @@ class SupportsHostProvisioning(Protocol): """Protocol interface for Host/VM provisioning operations.""" + # pylint: disable=unnecessary-ellipsis + def provision_host(self, params: dict) -> tuple["Status", dict]: """ Check if Host/VM is ready. Deploy a new Host/VM, if necessary. @@ -31,6 +33,7 @@ def provision_host(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def wait_host_deployment(self, params: dict, *, is_setup: bool) -> tuple["Status", dict]: """ @@ -52,6 +55,7 @@ def wait_host_deployment(self, params: dict, *, is_setup: bool) -> tuple["Status Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} Result is info on the operation runtime if SUCCEEDED, otherwise {}. """ + ... def deprovision_host(self, params: dict) -> tuple["Status", dict]: """ @@ -68,6 +72,7 @@ def deprovision_host(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def deallocate_host(self, params: dict) -> tuple["Status", dict]: """ @@ -88,3 +93,4 @@ def deallocate_host(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... diff --git a/mlos_bench/mlos_bench/services/types/local_exec_type.py b/mlos_bench/mlos_bench/services/types/local_exec_type.py index 12b8cafdf61..831870c194d 100644 --- a/mlos_bench/mlos_bench/services/types/local_exec_type.py +++ b/mlos_bench/mlos_bench/services/types/local_exec_type.py @@ -24,6 +24,9 @@ class SupportsLocalExec(Protocol): vs the target environment. Used in LocalEnv and provided by LocalExecService. """ + # Needed by pyright + # pylint: disable=unnecessary-ellipsis,redundant-returns-doc + def local_exec( self, script_lines: Iterable[str], @@ -49,6 +52,7 @@ def local_exec( (return_code, stdout, stderr) : (int, str, str) A 3-tuple of return code, stdout, and stderr of the script process. """ + ... def temp_dir_context( self, @@ -59,7 +63,7 @@ def temp_dir_context( Parameters ---------- - path : str + path : str | None A path to the temporary directory. Create a new one if None. Returns @@ -67,3 +71,4 @@ def temp_dir_context( temp_dir_context : tempfile.TemporaryDirectory Temporary directory context to use in the `with` clause. """ + ... diff --git a/mlos_bench/mlos_bench/services/types/network_provisioner_type.py b/mlos_bench/mlos_bench/services/types/network_provisioner_type.py index 58248ae486a..00f63a51419 100644 --- a/mlos_bench/mlos_bench/services/types/network_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/network_provisioner_type.py @@ -14,6 +14,8 @@ class SupportsNetworkProvisioning(Protocol): """Protocol interface for Network provisioning operations.""" + # pylint: disable=unnecessary-ellipsis + def provision_network(self, params: dict) -> tuple["Status", dict]: """ Check if Network is ready. Deploy a new Network, if necessary. @@ -31,6 +33,7 @@ def provision_network(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def wait_network_deployment(self, params: dict, *, is_setup: bool) -> tuple["Status", dict]: """ @@ -52,6 +55,7 @@ def wait_network_deployment(self, params: dict, *, is_setup: bool) -> tuple["Sta Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} Result is info on the operation runtime if SUCCEEDED, otherwise {}. """ + ... def deprovision_network( self, @@ -75,3 +79,4 @@ def deprovision_network( A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... diff --git a/mlos_bench/mlos_bench/services/types/os_ops_type.py b/mlos_bench/mlos_bench/services/types/os_ops_type.py index 43c00f7f84c..f3b3c127136 100644 --- a/mlos_bench/mlos_bench/services/types/os_ops_type.py +++ b/mlos_bench/mlos_bench/services/types/os_ops_type.py @@ -14,6 +14,8 @@ class SupportsOSOps(Protocol): """Protocol interface for Host/OS operations.""" + # pylint: disable=unnecessary-ellipsis + def shutdown(self, params: dict, force: bool = False) -> tuple["Status", dict]: """ Initiates a (graceful) shutdown of the Host/VM OS. @@ -31,6 +33,7 @@ def shutdown(self, params: dict, force: bool = False) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def reboot(self, params: dict, force: bool = False) -> tuple["Status", dict]: """ @@ -49,6 +52,7 @@ def reboot(self, params: dict, force: bool = False) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def wait_os_operation(self, params: dict) -> tuple["Status", dict]: """ @@ -69,3 +73,4 @@ def wait_os_operation(self, params: dict) -> tuple["Status", dict]: Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} Result is info on the operation runtime if SUCCEEDED, otherwise {}. """ + ... diff --git a/mlos_bench/mlos_bench/services/types/remote_config_type.py b/mlos_bench/mlos_bench/services/types/remote_config_type.py index 7a96e16fb40..09a1e8c20fe 100644 --- a/mlos_bench/mlos_bench/services/types/remote_config_type.py +++ b/mlos_bench/mlos_bench/services/types/remote_config_type.py @@ -14,6 +14,8 @@ class SupportsRemoteConfig(Protocol): """Protocol interface for configuring cloud services.""" + # pylint: disable=unnecessary-ellipsis + def configure(self, config: dict[str, Any], params: dict[str, Any]) -> tuple["Status", dict]: """ Update the parameters of a SaaS service in the cloud. @@ -31,6 +33,7 @@ def configure(self, config: dict[str, Any], params: dict[str, Any]) -> tuple["St A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def is_config_pending(self, config: dict[str, Any]) -> tuple["Status", dict]: """ @@ -49,3 +52,4 @@ def is_config_pending(self, config: dict[str, Any]) -> tuple["Status", dict]: If "isConfigPendingReboot" is set to True, rebooting a VM is necessary. Status is one of {PENDING, TIMED_OUT, SUCCEEDED, FAILED} """ + ... diff --git a/mlos_bench/mlos_bench/services/types/remote_exec_type.py b/mlos_bench/mlos_bench/services/types/remote_exec_type.py index d90134070b7..052ad9be7da 100644 --- a/mlos_bench/mlos_bench/services/types/remote_exec_type.py +++ b/mlos_bench/mlos_bench/services/types/remote_exec_type.py @@ -19,6 +19,8 @@ class SupportsRemoteExec(Protocol): on a remote host OS. """ + # pylint: disable=unnecessary-ellipsis + def remote_exec( self, script: Iterable[str], @@ -46,6 +48,7 @@ def remote_exec( A pair of Status and result. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def get_remote_exec_results(self, config: dict) -> tuple["Status", dict]: """ @@ -64,3 +67,4 @@ def get_remote_exec_results(self, config: dict) -> tuple["Status", dict]: A pair of Status and result. Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} """ + ... diff --git a/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py b/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py index e56be3d6083..a67e9f0274a 100644 --- a/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py +++ b/mlos_bench/mlos_bench/services/types/vm_provisioner_type.py @@ -14,6 +14,8 @@ class SupportsVMOps(Protocol): """Protocol interface for VM provisioning operations.""" + # pylint: disable=unnecessary-ellipsis + def vm_provision(self, params: dict) -> tuple["Status", dict]: """ Check if VM is ready. Deploy a new VM, if necessary. @@ -31,6 +33,7 @@ def vm_provision(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def wait_vm_deployment(self, is_setup: bool, params: dict) -> tuple["Status", dict]: """ @@ -51,6 +54,7 @@ def wait_vm_deployment(self, is_setup: bool, params: dict) -> tuple["Status", di Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} Result is info on the operation runtime if SUCCEEDED, otherwise {}. """ + ... def vm_start(self, params: dict) -> tuple["Status", dict]: """ @@ -67,6 +71,7 @@ def vm_start(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def vm_stop(self, params: dict) -> tuple["Status", dict]: """ @@ -83,6 +88,7 @@ def vm_stop(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def vm_restart(self, params: dict) -> tuple["Status", dict]: """ @@ -99,6 +105,7 @@ def vm_restart(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def vm_deprovision(self, params: dict) -> tuple["Status", dict]: """ @@ -115,6 +122,7 @@ def vm_deprovision(self, params: dict) -> tuple["Status", dict]: A pair of Status and result. The result is always {}. Status is one of {PENDING, SUCCEEDED, FAILED} """ + ... def wait_vm_operation(self, params: dict) -> tuple["Status", dict]: """ @@ -135,3 +143,4 @@ def wait_vm_operation(self, params: dict) -> tuple["Status", dict]: Status is one of {PENDING, SUCCEEDED, FAILED, TIMED_OUT} Result is info on the operation runtime if SUCCEEDED, otherwise {}. """ + ... diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py index f902571e9fa..62daa0232c2 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment.py @@ -293,9 +293,11 @@ def _get_config_id(self, conn: Connection, tunables: TunableGroups) -> int: if cur_config is not None: return int(cur_config.config_id) # mypy doesn't know it's always int # Config not found, create a new one: - config_id: int = conn.execute( + new_config_result = conn.execute( self._schema.config.insert().values(config_hash=config_hash) - ).inserted_primary_key[0] + ).inserted_primary_key + assert new_config_result + config_id: int = new_config_result[0] save_params( conn, self._schema.config_param, diff --git a/mlos_bench/mlos_bench/tests/environments/composite_env_service_test.py b/mlos_bench/mlos_bench/tests/environments/composite_env_service_test.py index 0d81ec78477..b8059a3f67e 100644 --- a/mlos_bench/mlos_bench/tests/environments/composite_env_service_test.py +++ b/mlos_bench/mlos_bench/tests/environments/composite_env_service_test.py @@ -10,6 +10,7 @@ from mlos_bench.environments.composite_env import CompositeEnv from mlos_bench.services.config_persistence import ConfigPersistenceService from mlos_bench.services.local.local_exec import LocalExecService +from mlos_bench.services.types.local_exec_type import SupportsLocalExec from mlos_bench.tunables.tunable_groups import TunableGroups from mlos_bench.util import path_join @@ -58,6 +59,7 @@ def test_composite_services(composite_env: CompositeEnv) -> None: for i, path in ((0, "_test_tmp_global"), (1, "_test_tmp_other_2"), (2, "_test_tmp_other_3")): service = composite_env.children[i]._service # pylint: disable=protected-access assert service is not None and hasattr(service, "temp_dir_context") + assert isinstance(service, SupportsLocalExec) with service.temp_dir_context() as temp_dir: assert os.path.samefile(temp_dir, path) os.rmdir(path) diff --git a/mlos_bench/mlos_bench/tests/event_loop_context_test.py b/mlos_bench/mlos_bench/tests/event_loop_context_test.py index cc6cd825b3d..7eff0fe7451 100644 --- a/mlos_bench/mlos_bench/tests/event_loop_context_test.py +++ b/mlos_bench/mlos_bench/tests/event_loop_context_test.py @@ -137,9 +137,15 @@ def test_event_loop_context() -> None: ): assert len(EventLoopContextCaller.EVENT_LOOP_CONTEXT._event_loop._ready) == 1 else: - assert len(EventLoopContextCaller.EVENT_LOOP_CONTEXT._event_loop._ready) == 0 + assert ( + len(EventLoopContextCaller.EVENT_LOOP_CONTEXT._event_loop._ready) # pyright: ignore + == 0 + ) assert hasattr(EventLoopContextCaller.EVENT_LOOP_CONTEXT._event_loop, "_scheduled") - assert len(EventLoopContextCaller.EVENT_LOOP_CONTEXT._event_loop._scheduled) == 0 + assert ( + len(EventLoopContextCaller.EVENT_LOOP_CONTEXT._event_loop._scheduled) # pyright: ignore + == 0 + ) with pytest.raises( AssertionError diff --git a/mlos_bench/mlos_bench/tests/launcher_run_test.py b/mlos_bench/mlos_bench/tests/launcher_run_test.py index 61c4679f027..4974a75b80b 100644 --- a/mlos_bench/mlos_bench/tests/launcher_run_test.py +++ b/mlos_bench/mlos_bench/tests/launcher_run_test.py @@ -83,9 +83,11 @@ def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecServ _launch_main_app( root_path, local_exec_service, - " --config cli/mock-bench.jsonc" - " --trial_config_repeat_count 5" - " --mock_env_seed -1", # Deterministic Mock Environment. + ( + " --config cli/mock-bench.jsonc" + " --trial_config_repeat_count 5" + " --mock_env_seed -1" # Deterministic Mock Environment. + ), [ f"^{_RE_DATE} run\\.py:\\d+ " + r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$", ], @@ -102,10 +104,12 @@ def test_launch_main_app_bench_values( _launch_main_app( root_path, local_exec_service, - " --config cli/mock-bench.jsonc" - " --tunable_values tunable-values/tunable-values-example.jsonc" - " --trial_config_repeat_count 5" - " --mock_env_seed -1", # Deterministic Mock Environment. + ( + " --config cli/mock-bench.jsonc" + " --tunable_values tunable-values/tunable-values-example.jsonc" + " --trial_config_repeat_count 5" + " --mock_env_seed -1" # Deterministic Mock Environment. + ), [ f"^{_RE_DATE} run\\.py:\\d+ " + r"_main INFO Final score: \{'score': 67\.11\d+\}\s*$", ], @@ -119,10 +123,12 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic _launch_main_app( root_path, local_exec_service, - "--config cli/mock-opt.jsonc" - " --trial_config_repeat_count 3" - " --max_suggestions 3" - " --mock_env_seed 42", # Noisy Mock Environment. + ( + "--config cli/mock-opt.jsonc" + " --trial_config_repeat_count 3" + " --max_suggestions 3" + " --mock_env_seed 42" # Noisy Mock Environment. + ), [ # Iteration 1: Expect first value to be the baseline f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " diff --git a/mlos_bench/mlos_bench/tests/services/mock_service.py b/mlos_bench/mlos_bench/tests/services/mock_service.py index 29cebc95387..868cd24712b 100644 --- a/mlos_bench/mlos_bench/tests/services/mock_service.py +++ b/mlos_bench/mlos_bench/tests/services/mock_service.py @@ -18,11 +18,15 @@ class SupportsSomeMethod(Protocol): """Protocol for some_method.""" + # pylint: disable=unnecessary-ellipsis + def some_method(self) -> str: """some_method.""" + ... def some_other_method(self) -> str: """some_other_method.""" + ... class MockServiceBase(Service, SupportsSomeMethod): diff --git a/mlos_bench/mlos_bench/tests/services/remote/mock/mock_vm_service.py b/mlos_bench/mlos_bench/tests/services/remote/mock/mock_vm_service.py index 62e5a2804a9..86a1e27deb1 100644 --- a/mlos_bench/mlos_bench/tests/services/remote/mock/mock_vm_service.py +++ b/mlos_bench/mlos_bench/tests/services/remote/mock/mock_vm_service.py @@ -17,6 +17,8 @@ class MockVMService(Service, SupportsHostProvisioning, SupportsHostOps, SupportsOSOps): """Mock VM service for testing.""" + # pylint: disable=too-many-ancestors + def __init__( self, config: dict[str, Any] | None = None, diff --git a/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py b/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py index a7b12c26921..3ce11ce82ba 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py @@ -27,6 +27,7 @@ def test_categorical_required_params() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -42,6 +43,7 @@ def test_categorical_weights() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) tunable = Tunable(name="test", config=config) assert tunable.weights == [25, 25, 50] @@ -57,6 +59,7 @@ def test_categorical_weights_wrong_count() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -72,6 +75,7 @@ def test_categorical_weights_wrong_values() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -87,6 +91,7 @@ def test_categorical_wrong_params() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -102,6 +107,7 @@ def test_categorical_disallow_special_values() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -173,6 +179,7 @@ def test_numerical_tunable_required_params(tunable_type: TunableValueTypeName) - }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name=f"test_{tunable_type}", config=config) @@ -188,6 +195,7 @@ def test_numerical_tunable_invalid_range(tunable_type: TunableValueTypeName) -> }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(AssertionError): Tunable(name=f"test_{tunable_type}", config=config) @@ -203,6 +211,7 @@ def test_numerical_tunable_reversed_range(tunable_type: TunableValueTypeName) -> }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name=f"test_{tunable_type}", config=config) @@ -221,6 +230,7 @@ def test_numerical_weights(tunable_type: TunableValueTypeName) -> None: }} """ config = json.loads(json_config) + assert isinstance(config, dict) tunable = Tunable(name="test", config=config) assert tunable.special == [0] assert tunable.weights == [0.1] @@ -239,6 +249,7 @@ def test_numerical_quantization(tunable_type: TunableValueTypeName) -> None: }} """ config = json.loads(json_config) + assert isinstance(config, dict) tunable = Tunable(name="test", config=config) expected = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] assert tunable.quantization_bins == len(expected) @@ -258,6 +269,7 @@ def test_numerical_log(tunable_type: TunableValueTypeName) -> None: }} """ config = json.loads(json_config) + assert isinstance(config, dict) tunable = Tunable(name="test", config=config) assert tunable.is_log @@ -274,6 +286,7 @@ def test_numerical_weights_no_specials(tunable_type: TunableValueTypeName) -> No }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -294,6 +307,7 @@ def test_numerical_weights_non_normalized(tunable_type: TunableValueTypeName) -> }} """ config = json.loads(json_config) + assert isinstance(config, dict) tunable = Tunable(name="test", config=config) assert tunable.special == [-1, 0] assert tunable.weights == [0, 10] # Zero weights are ok @@ -314,6 +328,7 @@ def test_numerical_weights_wrong_count(tunable_type: TunableValueTypeName) -> No }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -331,6 +346,7 @@ def test_numerical_weights_no_range_weight(tunable_type: TunableValueTypeName) - }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -348,6 +364,7 @@ def test_numerical_range_weight_no_weights(tunable_type: TunableValueTypeName) - }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -364,6 +381,7 @@ def test_numerical_range_weight_no_specials(tunable_type: TunableValueTypeName) }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -382,6 +400,7 @@ def test_numerical_weights_wrong_values(tunable_type: TunableValueTypeName) -> N }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -398,6 +417,7 @@ def test_numerical_quantization_wrong(tunable_type: TunableValueTypeName) -> Non }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) @@ -412,5 +432,6 @@ def test_bad_type() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test_bad_type", config=config) diff --git a/mlos_bench/mlos_bench/tests/tunables/tunable_distributions_test.py b/mlos_bench/mlos_bench/tests/tunables/tunable_distributions_test.py index 54f08e17092..2c8e0a6bfdc 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunable_distributions_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunable_distributions_test.py @@ -90,5 +90,6 @@ def test_numerical_distribution_unsupported(tunable_type: str) -> None: }} """ config = json.loads(json_config) + assert isinstance(config, dict) with pytest.raises(ValueError): Tunable(name="test", config=config) diff --git a/mlos_bench/mlos_bench/tests/tunables/tunable_slice_references_test.py b/mlos_bench/mlos_bench/tests/tunables/tunable_slice_references_test.py index 9d267d4e16c..e06f81f9ea2 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunable_slice_references_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunable_slice_references_test.py @@ -81,6 +81,7 @@ def test_overlapping_group_merge_tunable_groups(tunable_groups_config: dict) -> """ other_tunables_config = json.loads(other_tunables_json) + assert isinstance(other_tunables_config, dict) other_tunables = TunableGroups(other_tunables_config) with pytest.raises(ValueError): @@ -110,6 +111,7 @@ def test_bad_extended_merge_tunable_group(tunable_groups_config: dict) -> None: """ other_tunables_config = json.loads(other_tunables_json) + assert isinstance(other_tunables_config, dict) other_tunables = TunableGroups(other_tunables_config) with pytest.raises(ValueError): @@ -138,6 +140,7 @@ def test_good_extended_merge_tunable_group(tunable_groups_config: dict) -> None: """ other_tunables_config = json.loads(other_tunables_json) + assert isinstance(other_tunables_config, dict) other_tunables = TunableGroups(other_tunables_config) assert "new-param" not in parent_tunables diff --git a/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py b/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py index e4ddbed28be..77f7710b6d4 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py @@ -148,6 +148,7 @@ def test_tunable_assign_null_to_categorical() -> None: } """ config = json.loads(json_config) + assert isinstance(config, dict) categorical_tunable = Tunable(name="categorical_test", config=config) assert categorical_tunable assert categorical_tunable.category == "foo" diff --git a/mlos_bench/mlos_bench/tests/tunables/tunables_copy_test.py b/mlos_bench/mlos_bench/tests/tunables/tunables_copy_test.py index c5395fcb16f..87e97929d7d 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunables_copy_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunables_copy_test.py @@ -37,6 +37,8 @@ def test_copy_covariant_group(covariant_group: CovariantTunableGroup) -> None: new_value = [x for x in tunable.categories if x != tunable.category][0] elif tunable.is_numerical: new_value = tunable.numerical_value + 1 + else: + raise ValueError(f"{tunable=} :: unsupported tunable type.") covariant_group_copy[tunable] = new_value assert covariant_group_copy.is_updated() assert not covariant_group.is_updated() diff --git a/mlos_bench/mlos_bench/tunables/tunable.py b/mlos_bench/mlos_bench/tunables/tunable.py index fa08faa898c..69267bddcaf 100644 --- a/mlos_bench/mlos_bench/tunables/tunable.py +++ b/mlos_bench/mlos_bench/tunables/tunable.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. # """Tunable parameter definition.""" -import collections import copy import logging from collections.abc import Iterable, Sequence @@ -38,25 +37,40 @@ """Tunable value distribution type.""" -class DistributionDict(TypedDict, total=False): - """A typed dict for tunable parameters' distributions.""" +class DistributionDictOpt(TypedDict, total=False): + """ + A TypedDict for a :py:class:`.Tunable` parameter's optional ``distribution``'s + config parameters. + + Mostly used by type checking. These are the types expected to be received from the + json config. + """ - type: DistributionName params: dict[str, float] | None -class TunableDict(TypedDict, total=False): +class DistributionDict(DistributionDictOpt): """ - A typed dict for tunable parameters. + A TypedDict for a :py:class:`.Tunable` parameter's required ``distribution``'s + config parameters. - Mostly used for mypy type checking. + Mostly used by type checking. These are the types expected to be received from the + json config. + """ + + type: DistributionName - These are the types expected to be received from the json config. + +class TunableDictOpt(TypedDict, total=False): """ + A TypedDict for a :py:class:`.Tunable` parameter's optional config parameters. - type: TunableValueTypeName + Mostly used for mypy type checking. These are the types expected to be received from + the json config. + """ + + # Optional fields description: str | None - default: TunableValue values: list[str | None] | None range: Sequence[int] | Sequence[float] | None quantization_bins: int | None @@ -69,17 +83,64 @@ class TunableDict(TypedDict, total=False): meta: dict[str, Any] +class TunableDict(TunableDictOpt): + """ + A TypedDict for a :py:class:`.Tunable` parameter's required config parameters. + + Mostly used for mypy type checking. These are the types expected to be received from + the json config. + """ + + # Required fields + type: TunableValueTypeName + default: TunableValue + + +def tunable_dict_from_dict(config: dict[str, Any]) -> TunableDict: + """ + Creates a TunableDict from a regular dict. + + Parameters + ---------- + config : dict[str, Any] + A regular dict that represents a TunableDict. + + Returns + ------- + TunableDict + """ + _type = config.get("type") + if _type not in Tunable.DTYPE: + raise ValueError(f"Invalid parameter type: {_type}") + _meta = config.get("meta", {}) + return TunableDict( + type=_type, + description=config.get("description"), + default=config.get("default"), + values=config.get("values"), + range=config.get("range"), + quantization_bins=config.get("quantization_bins"), + log=config.get("log"), + distribution=config.get("distribution"), + special=config.get("special"), + values_weights=config.get("values_weights"), + special_weights=config.get("special_weights"), + range_weight=config.get("range_weight"), + meta=_meta, + ) + + class Tunable: # pylint: disable=too-many-instance-attributes,too-many-public-methods """A tunable parameter definition and its current value.""" - # Maps tunable types to their corresponding Python types by name. - _DTYPE: dict[TunableValueTypeName, TunableValueType] = { + DTYPE: dict[TunableValueTypeName, TunableValueType] = { "int": int, "float": float, "categorical": str, } + """Maps Tunable types to their corresponding Python types by name.""" - def __init__(self, name: str, config: TunableDict): + def __init__(self, name: str, config: dict): """ Create an instance of a new tunable parameter. @@ -95,25 +156,26 @@ def __init__(self, name: str, config: TunableDict): :py:mod:`mlos_bench.tunables` : for more information on tunable parameters and their configuration. """ + t_config = tunable_dict_from_dict(config) if not isinstance(name, str) or "!" in name: # TODO: Use a regex here and in JSON schema raise ValueError(f"Invalid name of the tunable: {name}") self._name = name - self._type: TunableValueTypeName = config["type"] # required - if self._type not in self._DTYPE: + self._type: TunableValueTypeName = t_config["type"] # required + if self._type not in self.DTYPE: raise ValueError(f"Invalid parameter type: {self._type}") - self._description = config.get("description") - self._default = config["default"] + self._description = t_config.get("description") + self._default = t_config["default"] self._default = self.dtype(self._default) if self._default is not None else self._default - self._values = config.get("values") + self._values = t_config.get("values") if self._values: self._values = [str(v) if v is not None else v for v in self._values] - self._meta: dict[str, Any] = config.get("meta", {}) + self._meta: dict[str, Any] = t_config.get("meta", {}) self._range: tuple[int, int] | tuple[float, float] | None = None - self._quantization_bins: int | None = config.get("quantization_bins") - self._log: bool | None = config.get("log") + self._quantization_bins: int | None = t_config.get("quantization_bins") + self._log: bool | None = t_config.get("log") self._distribution: DistributionName | None = None self._distribution_params: dict[str, float] = {} - distr = config.get("distribution") + distr = t_config.get("distribution") if distr: self._distribution = distr["type"] # required self._distribution_params = distr.get("params") or {} @@ -122,11 +184,11 @@ def __init__(self, name: str, config: TunableDict): assert len(config_range) == 2, f"Invalid range: {config_range}" config_range = (config_range[0], config_range[1]) self._range = config_range - self._special: list[int] | list[float] = config.get("special") or [] + self._special: list[int] | list[float] = t_config.get("special") or [] self._weights: list[float] = ( - config.get("values_weights") or config.get("special_weights") or [] + t_config.get("values_weights") or t_config.get("special_weights") or [] ) - self._range_weight: float | None = config.get("range_weight") + self._range_weight: float | None = t_config.get("range_weight") self._current_value = None self._sanity_check() self.value = self._default @@ -150,7 +212,7 @@ def _sanity_check_categorical(self) -> None: """ # pylint: disable=too-complex assert self.is_categorical - if not (self._values and isinstance(self._values, collections.abc.Iterable)): + if not (self._values and isinstance(self._values, Iterable)): raise ValueError(f"Must specify values for the categorical type tunable {self}") if self._range is not None: raise ValueError(f"Range must be None for the categorical type tunable {self}") @@ -523,7 +585,7 @@ def dtype(self) -> TunableValueType: dtype : type Data type of the tunable - one of {int, float, str}. """ - return self._DTYPE[self._type] + return self.DTYPE[self._type] @property def is_categorical(self) -> bool: diff --git a/mlos_bench/setup.py b/mlos_bench/setup.py index 1ff3dbc461e..7a3ccbc1021 100644 --- a/mlos_bench/setup.py +++ b/mlos_bench/setup.py @@ -25,7 +25,8 @@ warning(f"version.py not found, using dummy VERSION={VERSION}") try: - from setuptools_scm import get_version + # Note: setuptools_scm is typically only installed as a part of the build process. + from setuptools_scm import get_version # pyright: ignore[reportMissingImports] version = get_version(root="..", relative_to=__file__, fallback_version=VERSION) if version is not None: diff --git a/mlos_core/mlos_core/__init__.py b/mlos_core/mlos_core/__init__.py index cb19057a7d3..26d4adbf81c 100644 --- a/mlos_core/mlos_core/__init__.py +++ b/mlos_core/mlos_core/__init__.py @@ -111,7 +111,7 @@ franca for data science. - :py:meth:`mlos_core.optimizers.OptimizerFactory.create` is a factory function - that creates a new :py:type:`~mlos_core.optimizers.ConcreteOptimizer` instance + that creates a new :py:attr:`~mlos_core.optimizers.ConcreteOptimizer` instance To do this it uses the :py:class:`~mlos_core.optimizers.OptimizerType` enum to specify which underlying optimizer to use (e.g., diff --git a/mlos_core/mlos_core/optimizers/__init__.py b/mlos_core/mlos_core/optimizers/__init__.py index 9bec7a5ef8d..3b3786e1e30 100644 --- a/mlos_core/mlos_core/optimizers/__init__.py +++ b/mlos_core/mlos_core/optimizers/__init__.py @@ -29,7 +29,6 @@ """ from enum import Enum -from typing import TypeVar import ConfigSpace @@ -71,19 +70,9 @@ class will be used. """ -# To make mypy happy, we need to define a type variable for each optimizer type. -# https://github.com/python/mypy/issues/12952 -# ConcreteOptimizer = TypeVar('ConcreteOptimizer', *[member.value for member in OptimizerType]) -# To address this, we add a test for complete coverage of the enum. - -ConcreteOptimizer = TypeVar( - "ConcreteOptimizer", - RandomOptimizer, - FlamlOptimizer, - SmacOptimizer, -) +ConcreteOptimizer = RandomOptimizer | FlamlOptimizer | SmacOptimizer """ -Type variable for concrete optimizer classes. +Type alias for concrete optimizer classes. (e.g., :class:`~mlos_core.optimizers.bayesian_optimizers.smac_optimizer.SmacOptimizer`, etc.) """ @@ -108,7 +97,7 @@ def create( # pylint: disable=too-many-arguments optimizer_kwargs: dict | None = None, space_adapter_type: SpaceAdapterType = SpaceAdapterType.IDENTITY, space_adapter_kwargs: dict | None = None, - ) -> ConcreteOptimizer: # type: ignore[type-var] + ) -> ConcreteOptimizer: """ Create a new optimizer instance, given the parameter space, optimizer type, and potential optimizer options. diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index 910b75d7175..e39216f33a2 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -192,9 +192,11 @@ def __init__( initial_design_args["n_configs"] = n_random_init if n_random_init > 0.25 * max_trials and max_ratio is None: warning( - "Number of random initial configs (%d) is " - "greater than 25%% of max_trials (%d). " - "Consider setting max_ratio to avoid SMAC overriding n_random_init.", + ( + "Number of random initial configs (%d) is " + "greater than 25%% of max_trials (%d). " + "Consider setting max_ratio to avoid SMAC overriding n_random_init." + ), n_random_init, max_trials, ) diff --git a/mlos_core/mlos_core/spaces/adapters/__init__.py b/mlos_core/mlos_core/spaces/adapters/__init__.py index 16a2c3065fd..2cd29526400 100644 --- a/mlos_core/mlos_core/spaces/adapters/__init__.py +++ b/mlos_core/mlos_core/spaces/adapters/__init__.py @@ -32,7 +32,6 @@ """ from enum import Enum -from typing import TypeVar import ConfigSpace @@ -58,19 +57,8 @@ class SpaceAdapterType(Enum): """An instance of :class:`.LlamaTuneAdapter` class will be used.""" -# To make mypy happy, we need to define a type variable for each optimizer type. -# https://github.com/python/mypy/issues/12952 -# ConcreteSpaceAdapter = TypeVar( -# "ConcreteSpaceAdapter", -# *[member.value for member in SpaceAdapterType], -# ) -# To address this, we add a test for complete coverage of the enum. -ConcreteSpaceAdapter = TypeVar( - "ConcreteSpaceAdapter", - IdentityAdapter, - LlamaTuneAdapter, -) -"""Type variable for concrete SpaceAdapter classes (e.g., +ConcreteSpaceAdapter = IdentityAdapter | LlamaTuneAdapter +"""Type alias for concrete SpaceAdapter classes (e.g., :class:`~mlos_core.spaces.adapters.identity_adapter.IdentityAdapter`, etc.) """ @@ -86,9 +74,9 @@ class SpaceAdapterFactory: def create( *, parameter_space: ConfigSpace.ConfigurationSpace, - space_adapter_type: SpaceAdapterType = SpaceAdapterType.IDENTITY, + space_adapter_type: SpaceAdapterType | None = SpaceAdapterType.IDENTITY, space_adapter_kwargs: dict | None = None, - ) -> ConcreteSpaceAdapter: # type: ignore[type-var] + ) -> ConcreteSpaceAdapter: """ Create a new space adapter instance, given the parameter space and potential space adapter options. diff --git a/mlos_core/mlos_core/spaces/adapters/llamatune.py b/mlos_core/mlos_core/spaces/adapters/llamatune.py index 5dea393d765..73478de9a24 100644 --- a/mlos_core/mlos_core/spaces/adapters/llamatune.py +++ b/mlos_core/mlos_core/spaces/adapters/llamatune.py @@ -14,6 +14,7 @@ `_. """ import os +from typing import Any from warnings import warn import ConfigSpace @@ -565,7 +566,9 @@ def _try_generate_approx_inverse_mapping(self) -> None: # Compute pseudo-inverse matrix try: - inv_matrix: npt.NDArray = pinv(proj_matrix) + _inv = pinv(proj_matrix) + assert _inv is not None and not isinstance(_inv, tuple) + inv_matrix: npt.NDArray[np.floating[Any]] = _inv self._pinv_matrix = inv_matrix except LinAlgError as err: raise RuntimeError( diff --git a/mlos_core/mlos_core/spaces/converters/flaml.py b/mlos_core/mlos_core/spaces/converters/flaml.py index 7b203b42ad6..f82334bab98 100644 --- a/mlos_core/mlos_core/spaces/converters/flaml.py +++ b/mlos_core/mlos_core/spaces/converters/flaml.py @@ -12,14 +12,15 @@ import flaml.tune import flaml.tune.sample import numpy as np +from flaml.tune.sample import Domain if TYPE_CHECKING: from ConfigSpace.hyperparameters import Hyperparameter -FlamlDomain: TypeAlias = flaml.tune.sample.Domain +FlamlDomain: TypeAlias = Domain """Flaml domain type alias.""" -FlamlSpace: TypeAlias = dict[str, flaml.tune.sample.Domain] +FlamlSpace: TypeAlias = dict[str, Domain] """Flaml space type alias - a `dict[str, FlamlDomain]`""" diff --git a/mlos_core/mlos_core/spaces/converters/util.py b/mlos_core/mlos_core/spaces/converters/util.py index de0edb7cd1b..9b97a11f329 100644 --- a/mlos_core/mlos_core/spaces/converters/util.py +++ b/mlos_core/mlos_core/spaces/converters/util.py @@ -41,7 +41,11 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter: # No quantization requested. # Remove any previously applied patches. if hasattr(dist, "sample_vector_mlos_orig"): - setattr(dist, "sample_vector", dist.sample_vector_mlos_orig) + setattr( + dist, + "sample_vector", + dist.sample_vector_mlos_orig, # pyright: ignore[reportAttributeAccessIssue] + ) delattr(dist, "sample_vector_mlos_orig") return hp @@ -61,7 +65,10 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter: dist, "sample_vector", lambda n, *, seed=None: quantize( - dist.sample_vector_mlos_orig(n, seed=seed), + dist.sample_vector_mlos_orig( # pyright: ignore[reportAttributeAccessIssue] + n, + seed=seed, + ), bounds=(dist.lower_vectorized, dist.upper_vectorized), bins=quantization_bins, ), diff --git a/mlos_core/mlos_core/tests/__init__.py b/mlos_core/mlos_core/tests/__init__.py index 65f5d696866..d5856689615 100644 --- a/mlos_core/mlos_core/tests/__init__.py +++ b/mlos_core/mlos_core/tests/__init__.py @@ -6,7 +6,8 @@ from importlib import import_module from pkgutil import walk_packages -from typing import TypeAlias, TypeVar +from types import ModuleType +from typing import TypeVar # A common seed to use to avoid tracking down race conditions and intermingling # issues of seeds across tests that run in non-deterministic parallel orders. @@ -15,7 +16,7 @@ T = TypeVar("T") -def get_all_submodules(pkg: TypeAlias) -> list[str]: +def get_all_submodules(pkg: ModuleType) -> list[str]: """ Imports all submodules for a package and returns their names. diff --git a/mlos_core/mlos_core/tests/optimizers/optimizer_test.py b/mlos_core/mlos_core/tests/optimizers/optimizer_test.py index 2106ba64192..8913e976450 100644 --- a/mlos_core/mlos_core/tests/optimizers/optimizer_test.py +++ b/mlos_core/mlos_core/tests/optimizers/optimizer_test.py @@ -172,8 +172,7 @@ def objective(inp: float) -> pd.Series: ) def test_concrete_optimizer_type(optimizer_type: OptimizerType) -> None: """Test that all optimizer types are listed in the ConcreteOptimizer constraints.""" - # pylint: disable=no-member - assert optimizer_type.value in ConcreteOptimizer.__constraints__ + assert optimizer_type.value in ConcreteOptimizer.__args__ @pytest.mark.parametrize( diff --git a/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py b/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py index 0e65f815901..94cb64fe964 100644 --- a/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py +++ b/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py @@ -28,10 +28,11 @@ *list(SpaceAdapterType), ], ) -def test_concrete_optimizer_type(space_adapter_type: SpaceAdapterType) -> None: - """Test that all optimizer types are listed in the ConcreteOptimizer constraints.""" - # pylint: disable=no-member - assert space_adapter_type.value in ConcreteSpaceAdapter.__constraints__ +def test_concrete_space_adapter_type(space_adapter_type: SpaceAdapterType) -> None: + """Test that all spaceadapter types are listed in the ConcreteSpaceAdapter + constraints. + """ + assert space_adapter_type.value in ConcreteSpaceAdapter.__args__ @pytest.mark.parametrize( diff --git a/mlos_core/mlos_core/tests/spaces/spaces_test.py b/mlos_core/mlos_core/tests/spaces/spaces_test.py index 7e1b8d0d8fe..46d89e2ee9a 100644 --- a/mlos_core/mlos_core/tests/spaces/spaces_test.py +++ b/mlos_core/mlos_core/tests/spaces/spaces_test.py @@ -179,10 +179,10 @@ class TestFlamlConversion(BaseConversion): def sample( self, - config_space: FlamlSpace, # type: ignore[override] + config_space: OptimizerSpace, n_samples: int = 1, ) -> npt.NDArray: - assert isinstance(config_space, dict) + assert isinstance(config_space, dict) # FlamlSpace assert isinstance(next(iter(config_space.values())), flaml.tune.sample.Domain) ret: npt.NDArray = np.array( [domain.sample(size=n_samples) for domain in config_space.values()] diff --git a/mlos_core/setup.py b/mlos_core/setup.py index 81fe47c292e..b9ba6ac3c40 100644 --- a/mlos_core/setup.py +++ b/mlos_core/setup.py @@ -25,7 +25,8 @@ warning(f"version.py not found, using dummy VERSION={VERSION}") try: - from setuptools_scm import get_version + # Note: setuptools_scm is typically only installed as a part of the build process. + from setuptools_scm import get_version # pyright: ignore[reportMissingImports] version = get_version(root="..", relative_to=__file__, fallback_version=VERSION) if version is not None: diff --git a/mlos_viz/mlos_viz/base.py b/mlos_viz/mlos_viz/base.py index ca990fe33db..8360baf320d 100644 --- a/mlos_viz/mlos_viz/base.py +++ b/mlos_viz/mlos_viz/base.py @@ -493,7 +493,7 @@ def plot_top_n_configs( data=top_n_config_results_df, x=groupby_column, y=orderby_col, - legend=None, + legend=False, ax=axis, ) plt.grid() diff --git a/mlos_viz/setup.py b/mlos_viz/setup.py index c4c56ca9365..e646d549e92 100644 --- a/mlos_viz/setup.py +++ b/mlos_viz/setup.py @@ -25,7 +25,8 @@ warning(f"version.py not found, using dummy VERSION={VERSION}") try: - from setuptools_scm import get_version + # Note: setuptools_scm is typically only installed as a part of the build process. + from setuptools_scm import get_version # pyright: ignore[reportMissingImports] version = get_version(root="..", relative_to=__file__, fallback_version=VERSION) if version is not None: diff --git a/pyproject.toml b/pyproject.toml index a0504b034e7..69861825ae9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,4 +81,10 @@ exclude = [ "doc/build/html", "doc/build/doctrees", "htmlcov", + "mlos_*/build/", + "mlos_*/dist/", + "mlos_*/setup.py" ] +typeCheckingMode = "standard" +reportMissingTypeStubs = false # not as granular as mypy to override on a per package basis +#reportMissingImports = false # somewhat expected for build files and docs