diff --git a/.github/workflows/framework-tests.yaml b/.github/workflows/framework-tests.yaml index c3bdd4aa4..64ec4abdf 100644 --- a/.github/workflows/framework-tests.yaml +++ b/.github/workflows/framework-tests.yaml @@ -93,20 +93,27 @@ jobs: PEBBLE: /tmp/pebble pip-install: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Set up Python 3 uses: actions/setup-python@v2 with: - python-version: '3.11' + python-version: ${{ matrix.python-version }} + + - name: Install build dependencies + run: pip install wheel build - name: Build - run: python setup.py sdist + run: python -m build # Test that a pip install of the source dist .tar.gz will work - name: Test 'pip install' # Shouldn't happen, but pip install will fail if ls returns multiple lines run: pip install $(ls dist/ops*.gz) - diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2c7c13899..5067849ec 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,12 +19,10 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v1 - with: - python-version: "3.10" - - name: Install wheel - run: pip install wheel + - name: Install build dependencies + run: pip install wheel build - name: Build - run: python setup.py sdist bdist_wheel + run: python -m build - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.gitignore b/.gitignore index 34aed13b5..350428eb3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,7 @@ __pycache__ /sandbox -/build -/dist /ops.egg-info .idea -/docs/_build *~ .venv venv @@ -13,6 +10,11 @@ venv .coverage /.tox +# Build artifacts +/dist +/build +/docs/_build + # Smoke test artifacts *.tar.gz *.charm diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f18916664..2edd6f79d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -2,7 +2,10 @@ version: 2 python: install: - - requirements: docs/requirements.txt + - method: pip + path: . + extra_requirements: + - docs build: os: ubuntu-22.04 diff --git a/HACKING.md b/HACKING.md index a4ae822c7..632425bd3 100644 --- a/HACKING.md +++ b/HACKING.md @@ -180,26 +180,63 @@ next to the relevant content (e.g. headings, etc.). Noteworthy changes should also get a new entry in [CHANGES.md](CHANGES.md). +As noted above, you can generate a local copy of the API reference docs with tox: -## Dependencies +```sh +tox -e docs +open docs/_build/html/index.html +``` + +# Dependencies The Python dependencies of `ops` are kept as minimal as possible, to avoid bloat and to minimise conflict with the charm's dependencies. The dependencies -are listed in [requirements.txt](requirements.txt). +are listed in [pyproject.toml](pyproject.toml) in the `project.dependencies` section. + +# Dev Tools + +## Formatting and Checking +Test environments are managed with [tox](https://tox.wiki/) and executed with +[pytest](https://pytest.org), with coverage measured by +[coverage](https://coverage.readthedocs.io/). +Static type checking is done using [pyright](https://github.com/microsoft/pyright), +and extends the Python 3.8 type hinting support through the +[typing_extensions](https://pypi.org/project/typing-extensions/) package. + +Formatting uses [isort](https://pypi.org/project/isort/) and +[autopep8](https://pypi.org/project/autopep8/), with linting also using +[flake8](https://github.com/PyCQA/flake8), including the +[docstrings](https://pypi.org/project/flake8-docstrings/), +[builtins](https://pypi.org/project/flake8-builtins/) and +[pep8-naming](https://pypi.org/project/pep8-naming/) extensions. + +All tool configuration is kept in [project.toml](pyproject.toml). The list of +dependencies can be found in the relevant `tox.ini` environment `deps` field. + +## Building + +The build backend is [setuptools](https://pypi.org/project/setuptools/), and +the build frontend is [build](https://pypi.org/project/build/). # Publishing a Release To make a release of the ops library, do the following: -1. Visit the [releases page on GitHub](https://github.com/canonical/operator/releases). -2. Click "Draft a new release" -3. The "Release Title" is simply the full version number, in the form .. - E.g. 2.3.12 -4. Drop notes and a changelog in the description. -5. When you are ready, click "Publish". (If you are not ready, click "Save as Draft".) - -This will trigger an automatic build for the Python package and publish it to PyPI (the API token/secret is already set up in the repository settings). +1. Open a PR to change [version.py][ops/version.py]'s `version` to the + [appropriate string](https://semver.org/), and get that merged to main. +2. Visit the [releases page on GitHub](https://github.com/canonical/operator/releases). +3. Click "Draft a new release" +4. The "Release Title" is simply the full version number, in the form .. + and a brief summary of the main changes in the release + E.g. 2.3.12 Bug fixes for the Juju foobar feature when using Python 3.12 +5. Drop notes and a changelog in the description. +6. When you are ready, click "Publish". (If you are not ready, click "Save as Draft".) Wait for the new version to be published successfully to [the PyPI project](https://pypi.org/project/ops/). +7. Open a PR to change [version.py][ops/version.py]'s `version` to the expected + next version, with "+dev" appended (for example, if 3.14.1 is the next expected version, use + `'3.14.1.dev0'`). + +This will trigger an automatic build for the Python package and publish it to PyPI (authorization is handled via a [Trusted Publisher](https://docs.pypi.org/trusted-publishers/) relationship). See [.github/workflows/publish.yml](.github/workflows/publish.yml) for details. (Note that the versions in publish.yml refer to versions of the GitHub actions, not the versions of the ops library.) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..886f8e7a6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,10 @@ +graft ops +graft test + +include *.txt +include *.md +include MANIFEST.in +include pyproject.toml +include tox.ini + +global-exclude *~ *.py[cod] __pycache__ *.charm diff --git a/docs/requirements.in b/docs/requirements.in deleted file mode 100644 index f500c5e51..000000000 --- a/docs/requirements.in +++ /dev/null @@ -1,7 +0,0 @@ --r ../requirements.txt -sphinx==6.2.1 -sphinx-design -furo -sphinx-tabs -lxd-sphinx-extensions -sphinx-copybutton diff --git a/docs/requirements.txt b/docs/requirements.txt index 3862eb4c0..a52d53a20 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,31 +1,93 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --extra=docs --output-file=docs/requirements.txt pyproject.toml +# alabaster==0.7.13 -babel==2.12.1 + # via sphinx +babel==2.14.0 + # via sphinx beautifulsoup4==4.12.2 -certifi==2023.7.22 -charset-normalizer==3.1.0 + # via + # canonical-sphinx-extensions + # furo +canonical-sphinx-extensions==0.0.18 + # via lxd-sphinx-extensions +certifi==2023.11.17 + # via requests +charset-normalizer==3.3.2 + # via requests docutils==0.18.1 -furo==2023.5.20 -idna==3.4 + # via + # canonical-sphinx-extensions + # sphinx + # sphinx-tabs +furo==2023.9.10 + # via ops (pyproject.toml) +idna==3.6 + # via requests imagesize==1.4.1 -jinja2==3.1.2 -lxd-sphinx-extensions==0.0.7 -markupsafe==2.1.2 -packaging==23.1 -pygments==2.15.1 -pyyaml==6.0 + # via sphinx +importlib-metadata==7.0.1 + # via sphinx +jinja2==3.1.3 + # via sphinx +lxd-sphinx-extensions==0.0.16 + # via ops (pyproject.toml) +markupsafe==2.1.3 + # via jinja2 +packaging==23.2 + # via sphinx +pygments==2.17.2 + # via + # furo + # sphinx + # sphinx-tabs +pytz==2023.3.post1 + # via babel +pyyaml==6.0.1 + # via ops (pyproject.toml) requests==2.31.0 + # via + # canonical-sphinx-extensions + # sphinx snowballstemmer==2.2.0 -soupsieve==2.4.1 + # via sphinx +soupsieve==2.5 + # via beautifulsoup4 sphinx==6.2.1 -sphinx-basic-ng==1.0.0b1 + # via + # canonical-sphinx-extensions + # furo + # ops (pyproject.toml) + # sphinx-basic-ng + # sphinx-copybutton + # sphinx-design + # sphinx-tabs +sphinx-basic-ng==1.0.0b2 + # via furo sphinx-copybutton==0.5.2 -sphinx-design==0.4.1 -sphinx-tabs==3.4.1 + # via ops (pyproject.toml) +sphinx-design==0.5.0 + # via ops (pyproject.toml) +sphinx-tabs==3.4.4 + # via ops (pyproject.toml) sphinxcontrib-applehelp==1.0.4 + # via sphinx sphinxcontrib-devhelp==1.0.2 + # via sphinx sphinxcontrib-htmlhelp==2.0.1 + # via sphinx sphinxcontrib-jsmath==1.0.1 + # via sphinx sphinxcontrib-qthelp==1.0.3 + # via sphinx sphinxcontrib-serializinghtml==1.1.5 -urllib3==2.0.7 -websocket-client==1.5.2 + # via sphinx +urllib3==2.1.0 + # via requests +websocket-client==1.7.0 + # via ops (pyproject.toml) +zipp==3.17.0 + # via importlib-metadata diff --git a/docs/update_requirements.sh b/docs/update_requirements.sh deleted file mode 100755 index b59fc5574..000000000 --- a/docs/update_requirements.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Update and freeze docs/requirements.txt from docs/requirements.in - -set -ex - -python3 -m venv docsenv -source docsenv/bin/activate -pip install -r docs/requirements.in -pip freeze >docs/requirements.txt -deactivate -rm -rf docsenv diff --git a/ops/version.py b/ops/version.py index 79cc38106..94b18cb69 100644 --- a/ops/version.py +++ b/ops/version.py @@ -1,4 +1,4 @@ -# Copyright 2020 Canonical Ltd. +# Copyright 2023 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,45 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Helper to define the version of the ops library. +"""Package version. This module is NOT to be used when developing charms using ops. """ -import subprocess -from pathlib import Path - -__all__ = ('version',) - -_FALLBACK = '1.0' # this gets bumped after release - - -def _get_version(): - version = f"{_FALLBACK}.dev0+unknown" - - p = Path(__file__).parent - if (p.parent / '.git').exists(): - try: - proc = subprocess.run( - ['git', 'describe', '--tags', '--dirty'], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - cwd=p, - check=True, - encoding='utf-8') - except Exception: - pass - else: - version = proc.stdout.strip() - if '-' in version: - # version will look like -<#commits>-g[-dirty] - # in terms of PEP 440, the tag we'll make sure is a 'public version identifier'; - # everything after the first - needs to be a 'local version' - public, local = version.split('-', 1) - version = f"{public}+{local.replace('-', '.')}" - # version now +<#commits>.g[.dirty] - # which is PEP440-compliant (as long as is :-) - return version - - -version = _get_version() +version: str = '2.10.0.dev0' diff --git a/pyproject.toml b/pyproject.toml index fbf091025..1a55b3853 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,52 @@ +[project] +name = "ops" +description = "The Python library behind great charms" +readme = "README.md" +requires-python = ">=3.8" +authors = [ + {name="The Charm Tech team at Canonical Ltd.", email="charm-tech@lists.launchpad.com"}, +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", +] +dependencies = [ + "PyYAML==6.*", + "websocket-client==1.*", +] +dynamic = ["version"] + +[project.optional-dependencies] +docs = [ + "sphinx==6.2.1", + "sphinx-design", + "furo", + "sphinx-tabs", + "lxd-sphinx-extensions", + "sphinx-copybutton", +] + +[project.urls] +"Homepage" = "https://juju.is/docs/sdk" +"Repository" = "https://github.com/canonical/operator" +"Issues" = "https://github.com/canonical/operator/issues" +"Documentation" = "https://ops.readthedocs.io" +"Changelog" = "https://github.com/canonical/operator/blob/main/CHANGES.md" + +[build-system] +requires = [ + "setuptools>=60", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools.dynamic] +version = {attr = "ops.version.version"} + # Testing tools configuration [tool.coverage.run] branch = true diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 440f118ea..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,14 +0,0 @@ -isort~=5.11 -autopep8~=1.6 -flake8~=6.1 -flake8-docstrings~=1.7 -flake8-builtins~=2.1 -pyproject-flake8~=6.1 -pep8-naming~=0.13 -pytest~=7.2 -pyright==1.1.345 -pytest-operator~=0.23 -coverage[toml]~=7.0 -typing_extensions~=4.2 - --r requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index afd5bc095..000000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -PyYAML==6.* -websocket-client==1.* \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 5444a81b8..000000000 --- a/setup.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2019-2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Setup script for the ops library.""" - -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path - -from setuptools import find_packages, setup - - -def _read_me() -> str: - """Return the README content from the file.""" - with open("README.md", "rt", encoding="utf8") as fh: - readme = fh.read() - return readme - - -def _get_version() -> str: - """Get the version via ops/version.py, without loading ops/__init__.py.""" - spec = spec_from_file_location('ops.version', 'ops/version.py') - if spec is None: - raise ModuleNotFoundError('could not find /ops/version.py') - if spec.loader is None: - raise AttributeError('loader', spec, 'invalid module') - module = module_from_spec(spec) - spec.loader.exec_module(module) - - return module.version - - -version = _get_version() -version_path = Path("ops/version.py") -version_backup = Path("ops/version.py~") -version_backup.unlink(missing_ok=True) -version_path.rename(version_backup) -try: - with version_path.open("wt", encoding="utf8") as fh: - fh.write(f'''# this is a generated file - -version = {version!r} -''') - - setup( - name="ops", - version=version, - description="The Python library behind great charms", - long_description=_read_me(), - long_description_content_type="text/markdown", - license="Apache-2.0", - url="https://github.com/canonical/operator", - author="The Charmcraft team at Canonical Ltd.", - author_email="charmcraft@lists.launchpad.net", - packages=find_packages(include=('ops', 'ops.*')), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - ], - python_requires='>=3.8', - install_requires=[ # must stay in sync with requirements.txt (see test_install_requires) - 'PyYAML==6.*', - 'websocket-client==1.*', - ], - package_data={'ops': ['py.typed']}, - ) - -finally: - version_path.unlink() - version_backup.rename(version_path) diff --git a/test/test_infra.py b/test/test_infra.py index 47183f45d..aa734bace 100644 --- a/test/test_infra.py +++ b/test/test_infra.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ast import itertools import os import re @@ -22,12 +21,10 @@ import typing import unittest -import ops - def get_python_filepaths(include_tests: bool = True): """Helper to retrieve paths of Python files.""" - python_paths = ['setup.py'] + python_paths: typing.List[str] = [] roots = ['ops'] if include_tests: roots.append('test') @@ -70,50 +67,6 @@ def test_ensure_copyright(self): if issues: self.fail("Please add copyright headers to the following files:\n" + "\n".join(issues)) - def _run_setup(self, *args: str) -> str: - proc = subprocess.run( - (sys.executable, 'setup.py') + args, - stdout=subprocess.PIPE, - check=True, - encoding='utf-8') - return proc.stdout.strip() - - def test_setup_version(self): - setup_version = self._run_setup('--version') - - self.assertEqual(setup_version, ops.__version__) - - def test_setup_description(self): - with open("README.md", "rt", encoding="utf8") as fh: - disk_readme = fh.read().strip() - - setup_readme = self._run_setup('--long-description') - - self.assertEqual(setup_readme, disk_readme) - - def test_check(self): - self._run_setup('check', '--strict') - - def test_install_requires(self): - """Ensure that requirements.txt stays in sync with install_requires in setup.py.""" - with open('requirements.txt', encoding='utf-8') as f: - requirements = [line.strip() for line in f - if line.strip() and not line.startswith('#')] - - # For some reason "setup.py --requires" doesn't work, so do this the hard way - with open('setup.py', encoding='utf-8') as f: - lines: typing.List[str] = [] - for line in f: - if 'install_requires=[' in line: - break - for line in f: - if line.strip() == '],': - break - lines.append(line) - install_requires = ast.literal_eval('[' + '\n'.join(lines) + ']') - - self.assertEqual(requirements, install_requires) - class ImportersTestCase(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 186ca8bbf..049068235 100644 --- a/tox.ini +++ b/tox.ini @@ -25,15 +25,18 @@ passenv = [testenv:docs] description = Build the Sphinx docs -deps = - -r{toxinidir}/docs/requirements.txt +deps = pip-tools +commands_pre = + pip-compile --extra=docs -o docs/requirements.txt pyproject.toml + pip-sync {toxinidir}/docs/requirements.txt commands = sphinx-build -W --keep-going docs/ docs/_build/html [testenv:fmt] description = Apply coding style standards to code deps = - -r{toxinidir}/requirements-dev.txt + autopep8~=1.6 + isort~=5.11 commands = isort {[vars]all_path} autopep8 --in-place {[vars]all_path} @@ -41,7 +44,13 @@ commands = [testenv:lint] description = Check code against coding style standards deps = - -r{toxinidir}/requirements-dev.txt + autopep8~=1.6 + flake8~=6.1 + flake8-docstrings~=1.7 + flake8-builtins~=2.1 + isort~=5.11 + pep8-naming~=0.13 + pyproject-flake8~=6.1 commands = # pflake8 wrapper suppports config from pyproject.toml pflake8 {[vars]all_path} @@ -51,7 +60,11 @@ commands = [testenv:static] description = Run static type checker deps = - -r{toxinidir}/requirements-dev.txt + PyYAML==6.* + websocket-client==1.* + pyright==1.1.345 + pytest~=7.2 + typing_extensions~=4.2 commands = pyright {posargs} @@ -61,7 +74,11 @@ passenv = RUN_REAL_PEBBLE_TESTS PEBBLE deps = - -r{toxinidir}/requirements-dev.txt + PyYAML==6.* + websocket-client==1.* + coverage[toml]~=7.0 + pytest~=7.2 + typing_extensions~=4.2 commands = coverage run --source={[vars]src_path} \ -m pytest --ignore={[vars]tst_path}smoke -v --tb native {posargs} @@ -76,7 +93,11 @@ setenv = PEBBLE=/tmp/pebble RUN_REAL_PEBBLE_TESTS=1 deps = - -r{toxinidir}/requirements-dev.txt + PyYAML==6.* + websocket-client==1.* + coverage[toml]~=7.0 + pytest~=7.2 + typing_extensions~=4.2 commands = bash -c "umask 0; (pebble run --http=':4000' --create-dirs &>/dev/null & ) ; sleep 1; pytest -v --tb native test/test_real_pebble.py {posargs} ; killall -y 3m pebble" @@ -86,11 +107,14 @@ whitelist_externals = juju charmcraft bash deps = - -r{toxinidir}/requirements-dev.txt + build + coverage[toml]~=7.0 + pytest~=7.2 + pytest-operator~=0.23 commands = # Build a source tarball for ops, and drop it into the root directory of the smoke test charm. bash -c 'rm -vf ./test/charms/test_smoke/*.tar.gz # Cleanup old builds' - python {toxinidir}/setup.py sdist --dist-dir={toxinidir}/test/charms/test_smoke/ + python -m build --sdist --outdir={toxinidir}/test/charms/test_smoke/ # Inject the tarball into the smoke test charm's requirements. bash -c 'echo "./$(ls -1 ./test/charms/test_smoke/ | grep tar.gz)" > ./test/charms/test_smoke/requirements.txt'