diff --git a/.github/workflows/bump_version.yml b/.github/workflows/bump_version.yml deleted file mode 100644 index 1e59ffa..0000000 --- a/.github/workflows/bump_version.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Bump Version and Tag - -on: - push: - branches: [ main ] - paths-ignore: - - '**/_version.py' - - workflow_dispatch: - inputs: - bump_type: - description: 'Version bump type to use. If target_version is set, this will be ignored.' - type: choice - options: ['major', 'minor', 'patch', 'none'] - required: false - default: 'none' - target_version: - description: | - Optional target version (e.g. 1.2.3) to bump to. If not set, bump type will be used. - (leave empty for default) - required: false - default: '' - dry_run: - description: 'Dry Run' - type: boolean - required: false - default: false - -jobs: - get-version-info: - name: Get New Version Tag - runs-on: ubuntu-latest - if: github.actor != 'github-actions[bot]' - outputs: - version: ${{ steps.set-target-version.outputs.version || steps.version-tag.outputs.version }} - version_tag: ${{ steps.set-target-version.outputs.version_tag || steps.version-tag.outputs.version_tag }} - version_type: ${{ steps.set-target-version.outputs.version_type || steps.version-tag.outputs.version_type }} - previous_version: ${{ steps.get-current-version.outputs.previous_version || steps.version-tag.outputs.previous_version }} - update_required: ${{ steps.set-update-required.outputs.update_required }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get Bumped Version Tag - uses: ./.github/actions/bump-version-tag - id: version-tag - with: - bump_type: ${{ github.event.inputs.bump_type }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Print Version Info - # We only want to print if we are not a workflow call or if the target version is 'N/A' - if: github.event_name != 'workflow_dispatch' || !inputs.target_version - run: | - echo "Version: ${{ steps.version-tag.outputs.version }}" - echo "Version Tag: ${{ steps.version-tag.outputs.version_tag }}" - echo "Version Type: ${{ steps.version-tag.outputs.version_type }}" - echo "Previous Version: ${{ steps.version-tag.outputs.previous_version }}" - - name: Set Target Version - id: set-target-version - # We only want to set the target version if we are a workflow call and the target version is set - if: github.event_name == 'workflow_dispatch' && inputs.target_version - run: | - echo "Setting target version to ${{ inputs.target_version }}" - echo "version=${{ inputs.target_version }}" >> $GITHUB_OUTPUT - echo "version_tag=v${{ inputs.target_version }}" >> $GITHUB_OUTPUT - - name: Get Current Version - id: get-current-version - # We only want to get current version if we are a workflow call and the target version is set - if: github.event_name == 'workflow_dispatch' && inputs.target_version - uses: ./.github/actions/source-code-version-get - with: - version_file: _version.py - - name: Set Update Required - id: set-update-required - run: | - if [ "${{ steps.set-target-version.outputs.version || steps.version-tag.outputs.version }}" != "${{ steps.version-tag.outputs.previous_version }}" ]; then - echo "Update required" - echo "update_required=true" >> $GITHUB_OUTPUT - else - echo "No update required" - echo "update_required=false" >> $GITHUB_OUTPUT - fi - - update-version-and-tag: - name: Update Repo Tag and Version - runs-on: ubuntu-latest - needs: get-version-info - # We only want to run if: - # 1. We are not the GitHub bot - # 2. We are not a workflow call or we are not in dry run mode - # 3. The update is required (i.e. the version has changed) - if: | - github.actor != 'github-actions[bot]' && - (github.event_name != 'workflow_dispatch' || !inputs.dry_run) && - needs.get-version-info.outputs.update_required == 'true' - - steps: - - uses: actions/checkout@v4 - # This sets up the git user for the GitHub bot - - name: Configure Git User - run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - # This sets up ssh keys for the AllenInstitute GitHub - - name: Configure AllenInstitute Repo Authorization - uses: ./.github/actions/configure-org-repo-authorization - with: - token: ${{ secrets.AI_PACKAGES_TOKEN }} - ssh_private_key: ${{ secrets.AIBSGITHUB_PRIVATE_KEY }} - - name: Update Version - uses: ./.github/actions/source-code-version-update - with: - version: ${{ needs.get-version-info.outputs.version }} - version_tag: ${{ needs.get-version-info.outputs.version_tag }} - version_file: _version.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 82e4b7a..876cb16 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,37 +1,139 @@ -name: Create Release +name: Publish to PyPI on: - # push: - # tags: - # - 'v*' - workflow_dispatch: inputs: - tag: - description: 'Target Tag for Release' - type: string + part: + description: "Semver part to bump (major, minor, patch)" + type: choice required: true - force: - description: 'Force Release?' + default: "patch" + options: ["major", "minor", "patch"] + dry-run: + description: "Dry run" type: boolean - required: false - default: false + required: true + default: true + python-version: + description: "Python version used to build the distribution" + type: choice + required: true + default: "3.11" + options: ["3.9", "3.10", "3.11", "3.12"] jobs: - create-release: + bump: + name: Bump version runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + outputs: + VERSION: ${{ steps.get-version-and-commit-sha.outputs.VERSION }} + SHORT_VERSION: ${{ steps.get-version-and-commit-sha.outputs.SHORT_VERSION }} + MAJOR_VERSION: ${{ steps.get-version-and-commit-sha.outputs.MAJOR_VERSION }} + MINOR_VERSION: ${{ steps.get-version-and-commit-sha.outputs.MINOR_VERSION }} + PATCH_VERSION: ${{ steps.get-version-and-commit-sha.outputs.PATCH_VERSION }} + COMMIT_SHA: ${{ steps.get-version-and-commit-sha.outputs.COMMIT_SHA }} steps: - - name: Checkout code + - name: Checkout uses: actions/checkout@v4 + - name: Set up Python ${{ github.event.inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ github.event.inputs.python-version }} + cache: 'pip' + - name: Set up AllenInstitute Repo Authorization + uses: ./.github/actions/configure-org-repo-authorization + with: + token: ${{ secrets.AI_PACKAGES_TOKEN }} + - name: Get tags + run: git fetch --tags origin + - name: Configure git for github-actions[bot] + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Run Install + run: | + make install-release + shell: bash + - name: Bump version with bumpversion + run: | + source .venv/bin/activate + bump-my-version bump ${{ github.event.inputs.part }} + - name: Commit and push with tags + if: ${{ github.event.inputs.dry-run == 'false' }} + run: | + git push --follow-tags + - name: Get version and commit SHA + id: get-version-and-commit-sha + run: | + latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`) + # remove the leading v from tag + version="${latest_tag:1}" + echo "VERSION=$version" >> $GITHUB_OUTPUT + major_version="$(cut -d '.' -f 1 <<< $version)" + echo "MAJOR_VERSION=$major_version" >> $GITHUB_OUTPUT + minor_version="$(cut -d '.' -f 2 <<< $version)" + echo "MINOR_VERSION=$minor_version" >> $GITHUB_OUTPUT + patch_version="$(cut -d '.' -f 3 <<< $version)" + echo "PATCH_VERSION=$patch_version" >> $GITHUB_OUTPUT + short_version="$major_version.$minor_version" + echo "SHORT_VERSION=$short_version" >> $GITHUB_OUTPUT + commit_sha=$(git rev-parse $latest_tag) + echo "COMMIT_SHA=$commit_sha" >> $GITHUB_OUTPUT + - name: Show version + run: | + echo VERSION: ${{ steps.get-version-and-commit-sha.outputs.VERSION }} + echo SHORT_VERSION: ${{ steps.get-version-and-commit-sha.outputs.SHORT_VERSION }} + echo MAJOR_VERSION: ${{ steps.get-version-and-commit-sha.outputs.MAJOR_VERSION }} + echo MINOR_VERSION: ${{ steps.get-version-and-commit-sha.outputs.MINOR_VERSION }} + echo PATCH_VERSION: ${{ steps.get-version-and-commit-sha.outputs.PATCH_VERSION }} + echo COMMIT_SHA: ${{ steps.get-version-and-commit-sha.outputs.COMMIT_SHA }} + + build: + name: Build distribution + runs-on: ubuntu-latest + needs: bump + if: ${{ github.event.inputs.dry-run == 'false' }} + steps: + - uses: actions/checkout@v4 with: - ref: ${{ github.event.inputs.tag || github.ref }} - - name: Create Release with Changelog - id: create_release - uses: softprops/action-gh-release@v1 + ref: ${{ needs.bump.outputs.COMMIT_SHA }} + - name: Set up Python ${{ github.event.inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ github.event.inputs.python-version }} + cache: 'pip' + - name: Set up AllenInstitute Repo Authorization + uses: ./.github/actions/setup-ai-github-urls + with: + token: ${{ secrets.AI_PACKAGES_TOKEN }} + - name: Run Release + run: | + make dist + shell: bash + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: Publish to PyPI + needs: build + if: ${{ github.event.inputs.dry-run == 'false' }} + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/aibs-informatics-core + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 with: - tag_name: ${{ github.event.inputs.tag || github.ref_name }} - name: Release ${{ github.event.inputs.tag || github.ref_name }} - draft: false - prerelease: false - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5d89d0a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing to `aibs-informatics-core` + +Contributions are welcome and appreciated! + +## Types of Contributions + +### Reporting Bugs + +Report bugs to our [issues page](https://github.com/aibs-informatics-core/issues). + +If you are reporting a bug, please include: + +- Your operating system name and version. +- Any details about your local setup that might be helpful in troubleshooting. +- Detailed steps to reproduce the bug, in the form of a [minimal reproducible example](https://stackoverflow.com/help/minimal-reproducible-example). + +### Making Changes + +Look through the GitHub issues for bugs, features, and other requests. Most issues will have a label that can help you identify the type of issue. + +### Submitting Feedback + +The best way to send feedback is to [create an issue](https://github.com/aibs-informatics-core/issues/new) on GitHub. + +If you are proposing a feature: + +- Explain in detail how it would work. +- Keep the scope as narrow as possible, to make it easier to implement. +- Remember that while contributions are welcome, developer/maintainer time is limited. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..89605f8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,33 @@ +Allen Institute Software License – This software license is the 2-clause BSD +license plus a third clause that prohibits redistribution and use for +commercial purposes without further permission. + +Copyright © 2024. Allen Institute. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Redistributions and use for commercial purposes are not permitted without +the Allen Institute’s written permission. For purposes of this license, +commercial purposes are the incorporation of the Allen Institute's software +into anything for which you will charge fees or other compensation or use of +the software to perform a commercial service for a third party. Contact +terms@alleninstitute.org for commercial licensing opportunities. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/Makefile b/Makefile index a9bba92..fb15b75 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,10 @@ $(INSTALL_STAMP): $(PYTHON) $(DEP_FILES) $(PIP) install -e .[dev]; @touch $(INSTALL_STAMP) +install-release: clean-install-stamp $(PYTHON) $(DEP_FILES) ## Installs package for release + @. $(VENV_BIN)/activate;\ + $(PIP) install .[release] + install-force: clean-install-stamp install ## Force install package dependencies link-packages: ## Link local packages to virtualenv @@ -174,6 +178,10 @@ package: $(INSTALL_STAMP) ## Build package distribution ##@ Release Commands ##################### +dist: install-release ## Build source and wheel package + @. $(VENV_BIN)/activate;\ + $(PYTHON) -m build; + reinstall: obliterate install ## Recreate environment and install pre-build: obliterate ## Removes existing build environment diff --git a/README.md b/README.md index 518102e..6e595cb 100644 --- a/README.md +++ b/README.md @@ -4,25 +4,75 @@ --- -Core library to be used as foundation across multiple projects +## Overview +The AIBS Informatics Core library provides a collection of core functionalities and utilities for various projects at the Allen Institute for Brain Science. This library includes modules for handling environment configurations, data models, executors, and various utility functions. +## Modules -## Development +### Utils -*Move to separate DEVELOPMENT.md file later...* +The `utils` module provides various utility functions and classes to facilitate common tasks such as logging, hashing, and working with dictionaries and strings. -### Versioning +- **file_operations**: Functions for working with files and directories. +- **decorators**: Decorators for adding functionality to functions and methods. +- **hashing**: Functions for generating hashes. +- **json**: Functions for working with JSON data. +- +- **logging**: Utilities for setting up and managing logging. +- **tools.dicttools**: Functions for manipulating dictionaries. +- **tools.strtools**: Functions for manipulating strings. +- ** -This project uses [semantic versioning](https://semver.org/). The version is stored in the `_version.py` file. +### Models -This version is updated automatically via GitHub actions which use commit messages and PR titles to determine the version bump. The following keywords in a commit message will trigger a version bump: - - `(PATCH)` - for bug fixes, documentation updates, and small changes - - `(MINOR)` - for new features - - `(MAJOR)` - for breaking changes. +The `models` module defines protocols and base models used for serialization and deserialization of data. This module provides base classes for creating data models and utilities for working with data models. -Notes about versioning using the above keywords: - - The keyword must be in all caps and surrounded by parentheses - - The If you specify multiple keywords in a single commit message, the highest version bump will be used - - If you specify `(NONE)` keyword in a commit message, the version will not be bumped regardless of other keywords in the commit message. +There are a few base classes that can be used to create data models: +- **ModelBase**: A base class for creating data models. +- **DataClassModel**: A base class for creating data models using dataclasses. +- **SchemaModel**: A base class for creating data models using marshmallow schemas + dataclass. +- **WithValidation**: A mixin class for adding validation to data models. + +### Executors + +The `executors` module provides base classes and utilities for creating and running executors. Executors are responsible for handling specific tasks or requests. They allow for validating inputs/outputs based on schema data models + +- **BaseExecutor**: A base class for creating executors. +- **run_cli_executor**: A utility function for running executors from the command line. + +### Env + +The `env` module provides a concept of `EnvBase` which allows for creating isolated namespaces based on the type and name of environment: + +```python +env_base = EnvBase('dev-projectX') +env_base.prefixed('my_resource') # 'dev-projectX-my_resource' +``` + + +### Collections + +The `collections` module provides various collection classes and utilities for working with collections of data. +- Classes + - **DeepChainMap**: A class for creating recursive capable deep chain maps. + - **Tree**: A subclass of dict for creating tree structures from sequences. + - **ValidatedStr**: A class for creating validated strings based on regex patterns. +- Mixins + - **PostInitMixin**: A mixin class for handling post-initialization tasks. + - **EnvBaseMixins**: A mixin class for handling environment-related tasks. +- Enums + - **BaseEnum**: A base class for creating enums. + - **OrderedEnum**: A base class for creating ordered enums. + - **StrEnum**: A base class for creating string enums. + - **OrderedStrEnum**: A base class for creating ordered string enums. + + +## Contributing + +Any and all PRs are welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for more information. + +## Licensing + +This software is licensed under the Allen Institute Software License, which is the 2-clause BSD license plus a third clause that prohibits redistribution and use for commercial purposes without further permission. For more information, please visit [Allen Institute Terms of Use](https://alleninstitute.org/terms-of-use/). \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 14c41ec..3e70eb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,11 +11,13 @@ build-backend = "setuptools.build_meta" # ----------------------------------------------------------------------------- [project] name = "aibs-informatics-core" +authors = [{ name = "AIBS Informatics Group"}] +maintainers = [{ name = "AIBS Informatics Group"}] description = "Core package for informatics projects at the Allen Institute for Brain Science" -readme = "README.md" -dynamic = [ "version"] +readme = { file = "README.md", content-type = "text/markdown" } +license = { file = "LICENSE" } +dynamic = ["version"] requires-python = ">=3.9" - dependencies = [ "marshmallow~=3.22.0", "dataclasses-json~=0.6.7", @@ -29,7 +31,13 @@ dependencies = [ [project.optional-dependencies] dev = [ - "aibs-informatics-test-resources @ git+ssh://git@github.com/AllenInstitute/aibs-informatics-test-resources.git@main#egg=aibs_informatics_test_resources", + "aibs-informatics-test-resources", +] +release = [ + "build", + "bump-my-version", + "twine", + "wheel", ] [project.scripts] @@ -44,6 +52,13 @@ where = ["src"] [tool.setuptools.dynamic] version = {attr = "aibs_informatics_core._version.__version__"} +[project.urls] +Documentation = "https://.github.io/aibs-informatics-core/" +Homepage = "https://github.com/AllenInstitute/aibs-informatics-core/" +Issues = "https://github.com/AllenInstitute/aibs-informatics-core/issues" +Repository = "https://github.com/AllenInstitute/aibs-informatics-core/" + + # ----------------------------------------------------------------------------- ## Pyright Configurations # https://github.com/microsoft/pyright/blob/main/docs/configuration.md @@ -205,3 +220,29 @@ line_length = 99 profile = "black" src_paths = ["src", "test"] +# ----------------------------------------------------------------------------- +## bumpversion Configurations +# https://callowayproject.github.io/bump-my-version/ +# ----------------------------------------------------------------------------- + +[tool.bumpversion] +allow_dirty = false +commit = true +commit_args = "" +current_version = "0.0.4" +ignore_missing_version = false +message = "Bump version: {current_version} → {new_version}" +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" +regex = false +replace = "{new_version}" +search = "{current_version}" +serialize = ["{major}.{minor}.{patch}"] +sign_tags = false +tag = true +tag_message = "Bump version: {current_version} → {new_version}" +tag_name = "v{new_version}" + +[[tool.bumpversion.files]] +filename = "src/aibs_informatics_core/_version.py" +search = "__version__ = \"{current_version}\"" +replace = "__version__ = \"{new_version}\"" \ No newline at end of file diff --git a/src/aibs_informatics_core/utils/multiprocessing.py b/src/aibs_informatics_core/utils/multiprocessing.py index 2359a7d..df9b2f9 100644 --- a/src/aibs_informatics_core/utils/multiprocessing.py +++ b/src/aibs_informatics_core/utils/multiprocessing.py @@ -29,12 +29,7 @@ def apply_args_and_kwargs(fn, args: Iterable[Any], kwargs: Mapping[str, Any]): def _starmap_apply( - fn: Callable[ - [ - Any, - ], - U, - ], + fn: Callable[[List[Any]], U], args: Sequence[Any], kwargs: Mapping[str, Any], ) -> U: @@ -42,18 +37,13 @@ def _starmap_apply( def parallel_starmap( - callable: Callable[ - [ - Any, - ], - U, - ], + callable: Callable[[Any], U], arguments: Sequence[T], keyword_arguments: Optional[Union[Sequence[Mapping[str, Any]], Mapping[str, Any]]] = None, pool_class: Optional[Type[mp_pool.Pool]] = None, processes: Optional[int] = None, chunk_size: Optional[int] = None, - callback: Optional[Callable[[T], Any]] = None, + callback: Optional[Callable[[List[T]], Any]] = None, error_callback: Optional[Callable[[BaseException], None]] = None, ) -> List[U]: pool_class = pool_class or mp_pool.Pool