From 3983b37c3c41500ea2587221f2dbcc12575cfa0e Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 18:05:20 -0500 Subject: [PATCH 01/19] use pyproject.toml --- MANIFEST.in | 2 +- pyproject.toml | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 49 ------------------------- setup.py | 3 -- 4 files changed, 99 insertions(+), 53 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index ed9524d..d715f2e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include README.md include LICENSE -include hissw/templates/* \ No newline at end of file +include hissw/templates/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a934ec7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,98 @@ +[build-system] +requires = [ + "setuptools", + "setuptools_scm", + "wheel", +] +build-backend = 'setuptools.build_meta' + +[project] +name = "hissw" +requires-python = ">=3.9" +dependencies = [ + "jinja2", + "scipy", +] +description = "Seamlessly integrate SSWIDL code into your Python workflow" +readme = {file="README.md", content-type = "text/markdown"} +authors = [ + {name="Will Barnes", email="will.t.barnes@gmail.com"}, + {name="Bin Chen"}, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", +] +keywords = [ + "solar", + "sun", + "ssw", + "solar-physics", + "idl", + "sswidl", + "solarsoft", +] + +[project.urls] +Homepage = "https://github.com/wtbarnes/hissw/" +Documentation = "https://wtbarnes.github.io/hissw/" +Source = "https://github.com/wtbarnes/hissw/" + +[project.optional-dependencies] +astropy = ["astropy"] +test = ["pytest", "astropy"] +docs = ["mkdocs", "mkdocs-material"] + +[tool.setuptools.package-data] +hissw = ["templates/*.pro", "templates/*.sh"] + +[tool.codespell] +skip = "*.fts,*.fits,venv,*.pro,*.asdf,*.sh" + +[tool.ruff] +target-version = "py39" +line-length = 110 +exclude=[ + ".git,", + "__pycache__", + "build", + "hissw/version.py", +] +show-fixes = true +show-source = true + +select = [ + "E", + "F", + "W", + "UP", + "PT", + #"RET", + #"TID", + +] +extend-ignore = [ + # pycodestyle (E, W) + "E501", # LineTooLong # TODO! fix + + # pytest (PT) + "PT001", # Always use pytest.fixture() + "PT004", # Fixtures which don't return anything should have leading _ + "PT007", # Parametrize should be lists of tuples # TODO! fix + "PT011", # Too broad exception assert # TODO! fix + "PT023", # Always use () on pytest decorators +] + +[tool.ruff.per-file-ignores] +# Part of configuration, not a package. +"setup.py" = ["INP001"] +"conftest.py" = ["INP001"] + +"__init__.py" = ["E402", "F401", "F403"] +"test_*.py" = ["B011", "D", "E402", "PGH001", "S101"] + + +[tool.ruff.pydocstyle] +convention = "numpy" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 950724b..0000000 --- a/setup.cfg +++ /dev/null @@ -1,49 +0,0 @@ -[metadata] -name = hissw -description = Seamlessly integrate SSWIDL code into your Python workflow -long_description = file: README.md -long_description_content_type = text/markdown -author = Will Barnes -author_email = will.t.barnes@gmail.com -url = https://github.com/wtbarnes/hissw -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Science/Research - License :: OSI Approved :: MIT License - Programming Language :: Python :: 3 -keywords = - solar - sun - ssw - solar-physics - idl - sswidl - solarsoft - -project_urls = - Documentation = https://wtbarnes.github.io/hissw/ - Source = https://github.com/wtbarnes/hissw/ - -[options] -zip_safe = False -packages = find: -setup_requires = - setuptools_scm -include_package_data = True -python_requires = >=3.6 -install_requires = - jinja2 - scipy - -[options.extras_require] -astropy = - astropy -test = - pytest - astropy -docs = - mkdocs - mkdocs-material - -[options.package_data] -hissw = templates/* \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index d5d43d7..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup(use_scm_version=True) From bb9b3499b322bac272f5a640487e12cdfcbeb5ec Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 18:06:16 -0500 Subject: [PATCH 02/19] replace verbose flag with logger --- hissw/environment.py | 133 +++++++++++++++++++++++--------------- hissw/logger.py | 18 ++++++ hissw/tests/test_hissw.py | 31 +++++---- 3 files changed, 118 insertions(+), 64 deletions(-) create mode 100644 hissw/logger.py diff --git a/hissw/environment.py b/hissw/environment.py index 4984acc..86809fe 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -1,22 +1,23 @@ """ Build SSW scripts from Jinja 2 templates """ -import os import datetime +import os import pathlib +import stat import subprocess import tempfile import jinja2 from scipy.io import readsav -from .filters import string_list_filter -from .read_config import defaults -from .util import SSWIDLError, IDLLicenseError -from .filters import * +from hissw.filters import (force_double_precision_filter, log10_filter, + string_list_filter, units_filter) +from hissw.read_config import defaults +from hissw.util import IDLLicenseError, SSWIDLError -class Environment(object): +class Environment: """ Environment for running SSW and IDL scripts @@ -52,10 +53,21 @@ class Environment(object): def __init__(self, ssw_packages=None, ssw_paths=None, extra_paths=None, ssw_home=None, idl_home=None, filters=None, idl_only=False, header=None, footer=None): + # NOTE: Import here to avoid circular imports + from hissw import log + self.log = log + self.idl_only = idl_only + self.ssw_home = ssw_home + self.idl_home = idl_home self.ssw_packages = ssw_packages if ssw_packages is not None else [] self.ssw_paths = ssw_paths if ssw_paths is not None else [] self.extra_paths = extra_paths if extra_paths is not None else [] self.env = jinja2.Environment(loader=jinja2.PackageLoader('hissw', 'templates')) + self._setup_filters(filters) + self.header = '' if header is None else header + self.footer = '' if footer is None else footer + + def _setup_filters(self, filters): self.env.filters['to_unit'] = units_filter self.env.filters['log10'] = log10_filter self.env.filters['string_list'] = string_list_filter @@ -63,22 +75,30 @@ def __init__(self, ssw_packages=None, ssw_paths=None, extra_paths=None, if filters is not None: for k, v in filters.items(): self.env.filters[k] = v - self.header = '' if header is None else header - self.footer = '' if footer is None else footer - self._setup_home(ssw_home, idl_home, idl_only=idl_only) - def _setup_home(self, ssw_home, idl_home, idl_only=False): - """ - Setup SSW and IDL home locations - """ - self.ssw_home = defaults.get('ssw_home') if ssw_home is None else ssw_home - if idl_only: - self.ssw_home = None - else: - if self.ssw_home is None: + @property + def ssw_home(self): + return self._ssw_home + + @ssw_home.setter + def ssw_home(self, value): + self._ssw_home = defaults.get('ssw_home') if value is None else value + try: + self._ssw_home = pathlib.Path(self._ssw_home) + except TypeError: + if not self.idl_only: raise ValueError('ssw_home must be set at instantiation or in the hisswrc file.') - self.idl_home = defaults.get('idl_home') if idl_home is None else idl_home - if self.idl_home is None: + + @property + def idl_home(self): + return self._idl_home + + @idl_home.setter + def idl_home(self, value): + self._idl_home = defaults.get('idl_home') if value is None else value + try: + self._idl_home = pathlib.Path(self._idl_home) + except TypeError: raise ValueError('idl_home must be set at instantiation or in the hisswrc file.') @property @@ -86,17 +106,17 @@ def executable(self): """ Path to executable for running code """ - if self.ssw_home: - return 'sswidl' + if self.idl_only: + return self.idl_home / 'bin' / 'idl' else: - return os.path.join(self.idl_home, 'bin', 'idl') + return 'sswidl' def render_script(self, script, args): """ Render custom IDL scripts from templates and input arguments """ - if isinstance(script, (str, pathlib.Path)) and os.path.isfile(script): - with open(script, 'r') as f: + if isinstance(script, (str, pathlib.Path)) and pathlib.Path(script).is_file(): + with open(script) as f: script = f.read() if not isinstance(script, str): raise ValueError('Input script must either be a string or path to a script.') @@ -116,8 +136,6 @@ def procedure_script(self, script, save_vars, save_filename): """ Render inner procedure file """ - if save_vars is None: - save_vars = [] params = {'_script': script, '_save_vars': save_vars, '_save_filename': save_filename} @@ -143,7 +161,7 @@ def shell_script(self, command_filename): 'command_filename': command_filename} return self.env.get_template('startup.sh').render(**params) - def run(self, script, args=None, save_vars=None, verbose=True, **kwargs): + def run(self, script, args=None, save_vars=None, raise_exceptions=True): """ Set up the SSWIDL environment and run the supplied scripts. @@ -155,50 +173,59 @@ def run(self, script, args=None, save_vars=None, verbose=True, **kwargs): Input arguments to script save_vars : list, optional Variables to save and return from the IDL namespace - verbose : bool, optional - If True, print STDERR and SDOUT. Otherwise it will be - suppressed. This is useful for debugging. + raise_exceptions : bool, optional + If True, raise an exception based on the IDL output. This is left + as a flag because not raising an exception is sometimes useful + for debugging. """ + save_vars = [] if save_vars is None else save_vars args = {} if args is None else args # Expose the ssw_home variable in all scripts by default args.update({'ssw_home': self.ssw_home}) with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + date_string = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') # Get filenames - fn_template = os.path.join( - tmpdir, '{name}_'+datetime.datetime.now().strftime('%Y%m%d-%H%M%S')+'.{ext}') - save_filename = fn_template.format(name='idl_vars', ext='sav') - procedure_filename = fn_template.format(name='idl_procedure', ext='pro') - command_filename = fn_template.format(name='idl_script', ext='pro') - shell_filename = fn_template.format(name='ssw_shell', ext='sh') + save_filename = tmpdir / f'idl_vars_{date_string}.sav' + procedure_filename = tmpdir / f'idl_procedure_{date_string}.pro' + command_filename = tmpdir / f'idl_script_{date_string}.pro' + shell_filename = tmpdir / f'ssw_shell_{date_string}.sh' # Render and save scripts idl_script = self.custom_script(script, args) - with open(procedure_filename, 'w') as f: - f.write(self.procedure_script(idl_script, save_vars, save_filename)) - with open(command_filename, 'w') as f: - f.write(self.command_script(procedure_filename)) - with open(shell_filename, 'w') as f: - f.write(self.shell_script(command_filename,)) + files = [ + (procedure_filename, self.procedure_script(idl_script, save_vars, save_filename)), + (command_filename, self.command_script(procedure_filename)), + (shell_filename, self.shell_script(command_filename)), + ] + for filename, filescript in files: + self.log.debug(f'{filename}:\n{filescript}') + with open(filename, 'w') as f: + f.write(filescript) # Execute - subprocess.call(['chmod', 'u+x', shell_filename]) - cmd_output = subprocess.run([shell_filename], shell=True, stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - self._check_for_errors(cmd_output, verbose, **kwargs) + os.chmod(shell_filename, mode=stat.S_IRWXU) + cmd_output = subprocess.run( + [shell_filename], + shell=True, + capture_output=True, + text=True, + ) + self._check_for_errors(cmd_output, raise_exceptions=raise_exceptions) results = readsav(save_filename) return results - def _check_for_errors(self, output, verbose, **kwargs): + def _check_for_errors(self, output, raise_exceptions=True): """ Check IDL output to try and decide if an error has occurred """ - stdout = output.stdout.decode('utf-8') - stderr = output.stderr.decode('utf-8') + stdout = output.stdout + stderr = output.stderr # NOTE: For some reason, not only errors are output to stderr so we # have to check it for certain keywords to see if an error occurred - if kwargs.get('raise_exceptions', True): + if raise_exceptions: if 'execution halted' in stderr.lower(): raise SSWIDLError(stderr) if 'failed to acquire license' in stderr.lower(): raise IDLLicenseError(stderr) - if verbose: - print(f'{stderr}\n{stdout}') + self.log.warning(stderr) + self.log.info(stdout) diff --git a/hissw/logger.py b/hissw/logger.py new file mode 100644 index 0000000..00efdfb --- /dev/null +++ b/hissw/logger.py @@ -0,0 +1,18 @@ +import logging + +__all__ = ['_init_log'] + + +def _init_log(): + """ + Initializes the fiasco log. + + In most circumstances this is called automatically when importing + fiasco. This code is based on that provided by Astropy see + "licenses/ASTROPY.rst". + """ + orig_logger_cls = logging.getLoggerClass() + log = logging.getLogger('hissw') + logging.setLoggerClass(orig_logger_cls) + + return log diff --git a/hissw/tests/test_hissw.py b/hissw/tests/test_hissw.py index bcbe82e..22dadf0 100644 --- a/hissw/tests/test_hissw.py +++ b/hissw/tests/test_hissw.py @@ -1,14 +1,13 @@ """ Module level tests """ +import astropy.units as u +import numpy as np import pytest + import hissw -import numpy as np -import astropy.units as u from hissw.util import SSWIDLError -run_kwargs = {'verbose': True} - @pytest.fixture def idl_env(idl_home): @@ -28,7 +27,7 @@ def test_exception(idl_env): Test exception catching """ with pytest.raises(SSWIDLError): - _ = idl_env.run('foobar', **run_kwargs) + _ = idl_env.run('foobar') def test_no_args(idl_env): @@ -41,7 +40,7 @@ def test_no_args(idl_env): j = REBIN(TRANSPOSE(LINDGEN(n)), n, n) array = (i GE j) ''' - results = idl_env.run(script, **run_kwargs) + results = idl_env.run(script) assert results['array'].shape == (5, 5) @@ -51,15 +50,26 @@ def test_with_args(idl_env): """ script = ''' n = {{ n }} - i = REBIN(LINDGEN(n), n, n) + i = REBIN(LINDGEN(n), n, n) j = REBIN(TRANSPOSE(LINDGEN(n)), n, n) array = (i GE j) ''' n = 100 - results = idl_env.run(script, args={'n': n}, **run_kwargs) + results = idl_env.run(script, args={'n': n}) assert results['array'].shape == (n, n) +@pytest.mark.parametrize(('log_level', 'log_record_length'), [ + ('DEBUG', 5), + ('INFO', 2), + ('WARNING', 1), +]) +def test_logging(idl_env, caplog, log_level, log_record_length): + caplog.set_level(log_level) + _ = idl_env.run('print, "Hello World"') + assert len(caplog.records) == log_record_length + + def test_aia_response_functions(ssw_env): """ Compute AIA response functions using AIA packages in SSW @@ -75,7 +85,7 @@ def test_aia_response_functions(ssw_env): resp335 = response.a335.tresp ''' args = {'flags': ['temp', 'dn', 'timedepend_date', 'evenorm']} - results = ssw_env.run(script, args=args, **run_kwargs) + results = ssw_env.run(script, args=args) for c in [94, 131, 171, 193, 211, 335]: assert f'resp{c}' in results assert results[f'resp{c}'].shape == results['logt'].shape @@ -129,7 +139,7 @@ def test_default_ssw_var(ssw_env): foo = '{{ ssw_home }}' """ res = ssw_env.run(script) - assert res['foo'].decode('utf-8') == ssw_env.ssw_home + assert res['foo'].decode('utf-8') == str(ssw_env.ssw_home) def test_script_from_file(idl_env, tmp_path): @@ -214,4 +224,3 @@ def test_force_double_precision_filter_with_quantity(var, idl_env): result = idl_env.run('var = {{ var | to_unit("h") | force_double_precision }}', args={'var': var}) assert u.allclose(var.to_value('h'), result['var'], atol=0.0, rtol=0.0) - From 737fd46af2a0415ff89553f9310edf34143c1952 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 18:07:31 -0500 Subject: [PATCH 03/19] run ruff with pre-commit --- .github/workflows/deploy-docs.yml | 4 ++-- .pre-commit-config.yaml | 39 +++++++++++++++++++++++++++++++ README.md | 2 +- docs/examples/simple_example.md | 6 ++--- hissw/__init__.py | 5 +++- hissw/filters.py | 8 +++---- hissw/read_config.py | 3 +-- hissw/util.py | 4 +++- 8 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index c8a8e73..984db0a 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -8,7 +8,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2.3.1 - - name: Python + - name: Python uses: actions/setup-python@v2 with: python-version: 3.9 @@ -22,4 +22,4 @@ jobs: uses: JamesIves/github-pages-deploy-action@4.1.4 with: branch: gh-pages - folder: site \ No newline at end of file + folder: site diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..947e38c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,39 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.1.5" + hooks: + - id: ruff + args: ["--fix"] + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort + entry: isort + require_serial: true + language: python + types: + - python + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-ast + - id: check-case-conflict + - id: trailing-whitespace + exclude: ".*(.fits|.fts|.fit|.txt|.pro|.asdf|.sh)" + - id: check-yaml + - id: debug-statements + - id: check-added-large-files + - id: end-of-file-fixer + exclude: ".*(.fits|.fts|.fit|.txt|.pro|.asdf|.sh|.bib|tca.*)" + - id: mixed-line-ending + exclude: ".*(.fits|.fts|.fit|.txt|.bib|.asdf|.sh|tca.*)" + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + additional_dependencies: + - tomli diff --git a/README.md b/README.md index b2da8f2..09e90d6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ $ pip install hissw ``` which will automatically install the package and its dependencies. -hissw depends on the [Jinja2](http://jinja.pocoo.org/docs/dev/) and [scipy](https://docs.scipy.org/doc/) libraries. +hissw depends on the [Jinja2](http://jinja.pocoo.org/docs/dev/) and [scipy](https://docs.scipy.org/doc/) libraries. You can also install these manually with [conda](https://www.anaconda.com/download/) or from PyPI, i.e. `pip install `. Additionally, you'll need a local install of IDL and the [Solarsoft library](http://www.lmsal.com/solarsoft/). diff --git a/docs/examples/simple_example.md b/docs/examples/simple_example.md index b5107cf..9f63a6e 100644 --- a/docs/examples/simple_example.md +++ b/docs/examples/simple_example.md @@ -5,7 +5,7 @@ import hissw import matplotlib.pyplot as plt script = ''' n = 5 -i = REBIN(LINDGEN(n), n, n) +i = REBIN(LINDGEN(n), n, n) j = REBIN(TRANSPOSE(LINDGEN(n)), n, n) mask = (i GE j) ''' @@ -22,11 +22,11 @@ But what if we want to input the size of our upper triangular array with Python? ```python script = ''' n = {{ n }} -i = REBIN(LINDGEN(n), n, n) +i = REBIN(LINDGEN(n), n, n) j = REBIN(TRANSPOSE(LINDGEN(n)), n, n) mask = (i GE j) ''' results = ssw.run(script, args={'n': 100}) plt.imshow(results['mask']) ``` -![Example 2](../images/ex2.png) \ No newline at end of file +![Example 2](../images/ex2.png) diff --git a/hissw/__init__.py b/hissw/__init__.py index ee0c902..f8426a5 100644 --- a/hissw/__init__.py +++ b/hissw/__init__.py @@ -1,5 +1,8 @@ ''' hissw -- integrate SSW into Python workflows ''' -from .read_config import defaults from .environment import Environment +from .logger import _init_log +from .read_config import defaults + +log = _init_log() diff --git a/hissw/filters.py b/hissw/filters.py index 61b7882..8d8ab3a 100644 --- a/hissw/filters.py +++ b/hissw/filters.py @@ -29,7 +29,7 @@ def log10_filter(value): def string_list_filter(string_list): """ Double quote a list of strings. - + This is needed when passing in a list of strings to IDL as each string in the list will not be quoted when passed into the template. """ @@ -54,9 +54,9 @@ def force_double_precision_filter(value): str_list = [force_double_precision_filter(x) for x in value] # NOTE: this has to be done manually because each entry is formatted as a # string such that the division is not evaluated in Python. However, we - # want this to be inserted as an array of integer divison operations. + # want this to be inserted as an array of integer division operations. return f"[{','.join(str_list)}]" else: - # If it is neither an array or list, + # If it is neither an array or list, a, b = value.as_integer_ratio() - return f'{a}d / {b}d' \ No newline at end of file + return f'{a}d / {b}d' diff --git a/hissw/read_config.py b/hissw/read_config.py index 8f9e043..39bb97f 100644 --- a/hissw/read_config.py +++ b/hissw/read_config.py @@ -2,9 +2,8 @@ Read some default options if possible """ -import os import configparser - +import os defaults = {} defaults['hissw_home'] = os.path.join(os.environ['HOME'], '.hissw') diff --git a/hissw/util.py b/hissw/util.py index a81e9a8..11aeb4b 100644 --- a/hissw/util.py +++ b/hissw/util.py @@ -2,13 +2,15 @@ Any utility functions """ +__all__ = ['SSWIDLError', 'IDLLicenseError'] + class SSWIDLError(Exception): """ An error to raise when something goes wrong in SSW """ pass - + class IDLLicenseError(Exception): """ From ded837b8d155274342f2437f9b5cd00aaebc8b56 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 20:21:05 -0500 Subject: [PATCH 04/19] fix build to use pyproject.toml --- .github/workflows/release-pypi.yml | 6 +++--- pyproject.toml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml index 7bb0649..63e518d 100644 --- a/.github/workflows/release-pypi.yml +++ b/.github/workflows/release-pypi.yml @@ -17,12 +17,12 @@ jobs: python-version: '3.x' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine + python -m pip install --upgrade build + python -m pip install --upgrade twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist + python -m build twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml index a934ec7..f539751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ build-backend = 'setuptools.build_meta' [project] name = "hissw" +dynamic = ["version"] requires-python = ">=3.9" dependencies = [ "jinja2", @@ -45,9 +46,14 @@ astropy = ["astropy"] test = ["pytest", "astropy"] docs = ["mkdocs", "mkdocs-material"] +[tool.setuptools.packages.find] +where = ["hissw"] + [tool.setuptools.package-data] hissw = ["templates/*.pro", "templates/*.sh"] +[tool.setuptools_scm] + [tool.codespell] skip = "*.fts,*.fits,venv,*.pro,*.asdf,*.sh" From 4e7d4e4796f4476ee471ac34c52c45a8cba23066 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 20:21:50 -0500 Subject: [PATCH 05/19] add additional exceptions for catching missing SSW/IDL install --- hissw/environment.py | 9 ++++++- hissw/templates/parent.pro | 2 ++ hissw/templates/startup.sh | 2 +- hissw/tests/test_hissw.py | 53 +++++++++++++++++++++++++++----------- hissw/util.py | 14 ++++++++++ 5 files changed, 63 insertions(+), 17 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index 86809fe..44eda92 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -14,7 +14,8 @@ from hissw.filters import (force_double_precision_filter, log10_filter, string_list_filter, units_filter) from hissw.read_config import defaults -from hissw.util import IDLLicenseError, SSWIDLError +from hissw.util import (IDLLicenseError, IDLNotFoundError, SSWIDLError, + SSWNotFoundError) class Environment: @@ -146,6 +147,7 @@ def command_script(self, procedure_filename): Generate parent IDL script """ params = {'ssw_paths': self.ssw_paths, + 'use_ssw': not self.idl_only, 'extra_paths': self.extra_paths, 'procedure_filename': procedure_filename} return self.env.get_template('parent.pro').render(**params) @@ -158,6 +160,7 @@ def shell_script(self, command_filename): 'ssw_home': self.ssw_home, 'ssw_packages': self.ssw_packages, 'idl_home': self.idl_home, + 'use_ssw': not self.idl_only, 'command_filename': command_filename} return self.env.get_template('startup.sh').render(**params) @@ -227,5 +230,9 @@ def _check_for_errors(self, output, raise_exceptions=True): raise SSWIDLError(stderr) if 'failed to acquire license' in stderr.lower(): raise IDLLicenseError(stderr) + if 'setup.ssw: no such file or directory' in stderr.lower(): + raise SSWNotFoundError(f'No SSW installation found at {self.ssw_home}.') + if 'idl: command not found' in stderr.lower(): + raise IDLNotFoundError(f'No IDL installation found at {self.idl_home}.') self.log.warning(stderr) self.log.info(stdout) diff --git a/hissw/templates/parent.pro b/hissw/templates/parent.pro index e6cef00..fca54e9 100644 --- a/hissw/templates/parent.pro +++ b/hissw/templates/parent.pro @@ -1,7 +1,9 @@ ;include paths for any packages we are loading +{% if use_ssw %} {% if ssw_paths | length > 0 %} ssw_path,/{{ ssw_paths | join(', /') }} {% endif %} +{% endif %} ;add any other paths we need to the IDL path !PATH={%for p in extra_paths%}EXPAND_PATH('{{ p }}')+':'+{%endfor%}!PATH ;run user scripts diff --git a/hissw/templates/startup.sh b/hissw/templates/startup.sh index 70ed1a5..ae9f99b 100644 --- a/hissw/templates/startup.sh +++ b/hissw/templates/startup.sh @@ -4,7 +4,7 @@ setenv SSW {{ ssw_home }} # Set SSW instruments setenv SSW_INSTR "{{ ssw_packages | join(' ') }}" # Setup needed environment variables -{% if ssw_home is not none %} +{% if use_ssw %} source $SSW/gen/setup/setup.ssw {% endif %} # Setup IDL environment diff --git a/hissw/tests/test_hissw.py b/hissw/tests/test_hissw.py index 22dadf0..69a041f 100644 --- a/hissw/tests/test_hissw.py +++ b/hissw/tests/test_hissw.py @@ -6,23 +6,34 @@ import pytest import hissw -from hissw.util import SSWIDLError +from hissw.util import IDLNotFoundError, SSWIDLError, SSWNotFoundError @pytest.fixture def idl_env(idl_home): - return hissw.Environment(idl_home=idl_home, idl_only=True) - + env = hissw.Environment(idl_home=idl_home, idl_only=True) + try: + _ = env.run('') + except IDLNotFoundError: + pytest.skip(f'Skipping IDL tests. No IDL installation found at {env.idl_home}.') + else: + return env @pytest.fixture -def ssw_env(idl_home, ssw_home): - return hissw.Environment(idl_home=idl_home, - ssw_home=ssw_home, - ssw_packages=['sdo/aia'], - ssw_paths=['aia']) - - -def test_exception(idl_env): +def ssw_env(idl_env, ssw_home): + env = hissw.Environment(idl_home=idl_env.idl_home, + ssw_home=ssw_home, + ssw_packages=['sdo/aia'], + ssw_paths=['aia']) + try: + _ = env.run('') + except SSWNotFoundError: + pytest.skip(f'Skipping SSW tests. No SSW installation found at {env.ssw_home}.') + else: + return env + + +def test_exception_idl_command(idl_env): """ Test exception catching """ @@ -30,6 +41,18 @@ def test_exception(idl_env): _ = idl_env.run('foobar') +def test_exception_missing_ssw(tmp_path): + env = hissw.Environment(ssw_home=tmp_path) + with pytest.raises(SSWNotFoundError): + _ = env.run('') + + +def test_exception_missing_idl(tmp_path): + env = hissw.Environment(idl_home=tmp_path, idl_only=True) + with pytest.raises(IDLNotFoundError): + _ = env.run('') + + def test_no_args(idl_env): """ No input arguments and no calls to SSW functions @@ -153,11 +176,11 @@ def test_script_from_file(idl_env, tmp_path): assert result['foo'] == (a + b) -def test_custom_filters(idl_home): +def test_custom_filters(idl_env): filters = { 'my_filter': lambda x: 'foo' if x < 0.5 else 'bar' } - env = hissw.Environment(idl_home=idl_home, idl_only=True, filters=filters) + env = hissw.Environment(idl_home=idl_env.idl_home, idl_only=True, filters=filters) script = ''' a = '{{ a | my_filter }}' b = '{{ b | my_filter }}' @@ -173,10 +196,10 @@ def test_invalid_script(idl_env): _ = idl_env.run(None) -def test_custom_header_footer(idl_home): +def test_custom_header_footer(idl_env): header = 'foo = {{ a }}' footer = 'bar = {{ a }} + {{ b }}' - env_custom = hissw.Environment(idl_home=idl_home, idl_only=True, + env_custom = hissw.Environment(idl_home=idl_env.idl_home, idl_only=True, header=header, footer=footer) script = ''' print, {{ a }} diff --git a/hissw/util.py b/hissw/util.py index 11aeb4b..167d60d 100644 --- a/hissw/util.py +++ b/hissw/util.py @@ -12,8 +12,22 @@ class SSWIDLError(Exception): pass +class SSWNotFoundError(Exception): + """ + An error to raise when an SSW installation cannot be found + """ + pass + + class IDLLicenseError(Exception): """ An error to raise when IDL cannot find a license """ pass + + +class IDLNotFoundError(Exception): + """ + An error to raise when an IDL installation cannot be found + """ + pass From 518deba063103a7c89ef76d34c42b8e1128fd5e8 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 20:37:16 -0500 Subject: [PATCH 06/19] use OA CI templates --- .github/workflows/release-pypi.yml | 28 -------------- .github/workflows/test.yml | 60 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 28 deletions(-) delete mode 100644 .github/workflows/release-pypi.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml deleted file mode 100644 index 63e518d..0000000 --- a/.github/workflows/release-pypi.yml +++ /dev/null @@ -1,28 +0,0 @@ -# A workflow to automatically publish the package when a new version is -# created -name: Upload Python Package - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade build - python -m pip install --upgrade twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python -m build - twine upload dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ffeb6a3 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,60 @@ +name: CI + +on: + push: + branches: + - 'main' + - '*.*' + tags: + - 'v*' + pull_request: + workflow_dispatch: + +jobs: + test: + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + with: + toxdeps: tox-pypi-filter + envs: | + - macos: py311 + - windows: py311 + - linux: py39 + - linux: py310 + - linux: py311 + codestyle: + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + with: + toxdeps: tox-pypi-filter + envs: | + - linux: codestyle + python-version: '3.11' + docs: + needs: [test] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + with: + toxdeps: tox-pypi-filter + envs: | + - linux: build_docs + python-version: '3.11' + publish: + # Build wheels when pushing to any branch except main + # publish.yml will only publish if tagged ^v.* + if: | + ( + github.event_name != 'pull_request' && ( + github.ref_name != 'main' || + github.event_name == 'workflow_dispatch' + ) + ) || ( + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'Run publish') + ) + needs: [test] + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@main + with: + test_extras: 'dev' + test_command: 'pytest -p no:warnings --doctest-rst --pyargs hissw' + submodules: false + python-version: '3.11' + secrets: + pypi_token: ${{ secrets.PYPI_TOKEN }} From 9e749d82fb21b659371fd3d56f6f2df793cace58 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 20:37:31 -0500 Subject: [PATCH 07/19] tox --- tox.ini | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..161c941 --- /dev/null +++ b/tox.ini @@ -0,0 +1,68 @@ +[tox] +envlist = + py{39,310,311} + build_docs + codestyle +isolated_build = true +requires = + setuptools >= 30.3.0 + pip >= 19.3.1 + tox-pypi-filter >= 0.10 + +[testenv] + +# The following option combined with the use of the tox-pypi-filter above allows +# project-wide pinning of dependencies, e.g. if new versions of pytest do not +# work correctly with pytest-astropy plugins. Most of the time the pinnings file +# should be empty. +pypi_filter_requirements = https://raw.githubusercontent.com/sunpy/sunpy/main/.test_package_pins.txt + +# Pass through the following environment variables which may be needed for the CI +# Pass through the following environment variables which may be needed for the CI +passenv = + HOME + WINDIR + LC_ALL + LC_CTYPE + CC + CI + TRAVIS + +# Run the tests in a temporary directory to make sure that we don't import +# the package from the source tree +changedir = .tmp/{envname} + +# tox environments are constructed with so-called 'factors' (or terms) +# separated by hyphens, e.g. test-devdeps-cov. Lines below starting with factor: +# will only take effect if that factor is included in the environment name. To +# see a list of example environments that can be run, along with a description, +# run: +# +# tox -l -v +# +description = + run tests + +deps = +# The following indicates which extras_require from setup.cfg will be installed +extras = + test +commands = + pytest --pyargs pydrad {toxinidir}/docs --cov pydrad --cov-report=xml --cov-config={toxinidir}/setup.cfg {posargs} + + +[testenv:build_docs] +description = invoke mkdocs to build html docs +extras = docs +commands = + mkdocs build -d site + + +[testenv:codestyle] +skip_install = true +description = Run all style and file checks with pre-commit +deps = + pre-commit +commands = + pre-commit install-hooks + pre-commit run --all-files From b6f47f3a21a6e9482f3cce943265938a2906eeab Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Wed, 3 Jan 2024 20:47:18 -0500 Subject: [PATCH 08/19] fix tox config --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 161c941..b55c270 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ deps = extras = test commands = - pytest --pyargs pydrad {toxinidir}/docs --cov pydrad --cov-report=xml --cov-config={toxinidir}/setup.cfg {posargs} + pytest --pyargs hissw {posargs} [testenv:build_docs] From 1ca1bddeed216201e33280ebe0012a2580eebbd6 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Thu, 4 Jan 2024 11:02:48 -0500 Subject: [PATCH 09/19] add setup.py back to fix tox --- .github/workflows/test.yml | 2 +- setup.py | 3 +++ tox.ini | 8 +------- 3 files changed, 5 insertions(+), 8 deletions(-) create mode 100644 setup.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffeb6a3..7999d50 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,7 +53,7 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@main with: test_extras: 'dev' - test_command: 'pytest -p no:warnings --doctest-rst --pyargs hissw' + test_command: 'pytest -p no:warnings --pyargs hissw' submodules: false python-version: '3.11' secrets: diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d5d43d7 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup(use_scm_version=True) diff --git a/tox.ini b/tox.ini index b55c270..fdcea4a 100644 --- a/tox.ini +++ b/tox.ini @@ -11,12 +11,6 @@ requires = [testenv] -# The following option combined with the use of the tox-pypi-filter above allows -# project-wide pinning of dependencies, e.g. if new versions of pytest do not -# work correctly with pytest-astropy plugins. Most of the time the pinnings file -# should be empty. -pypi_filter_requirements = https://raw.githubusercontent.com/sunpy/sunpy/main/.test_package_pins.txt - # Pass through the following environment variables which may be needed for the CI # Pass through the following environment variables which may be needed for the CI passenv = @@ -48,7 +42,7 @@ deps = extras = test commands = - pytest --pyargs hissw {posargs} + python -m pytest --pyargs hissw {posargs} [testenv:build_docs] From 933fc055b3d0b6f7984f93a0e339af438fb27fa5 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Thu, 4 Jan 2024 11:20:37 -0500 Subject: [PATCH 10/19] make sure tests are skipped instead of erroring --- .github/workflows/test.yml | 3 +++ hissw/tests/test_hissw.py | 2 +- tox.ini | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7999d50..3a552d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,9 @@ jobs: test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: + # NOTE: These are purposefully set to meaningless directories where there is no SSW + # or IDL installation so that the relevant tests will be skipped. + posargs: '--ssw-home=/home/foo --idl-home=/home/bar' toxdeps: tox-pypi-filter envs: | - macos: py311 diff --git a/hissw/tests/test_hissw.py b/hissw/tests/test_hissw.py index 69a041f..c719f9e 100644 --- a/hissw/tests/test_hissw.py +++ b/hissw/tests/test_hissw.py @@ -42,7 +42,7 @@ def test_exception_idl_command(idl_env): def test_exception_missing_ssw(tmp_path): - env = hissw.Environment(ssw_home=tmp_path) + env = hissw.Environment(idl_home=tmp_path, ssw_home=tmp_path) with pytest.raises(SSWNotFoundError): _ = env.run('') diff --git a/tox.ini b/tox.ini index fdcea4a..33065c9 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,7 @@ commands = [testenv:build_docs] +changedir = . description = invoke mkdocs to build html docs extras = docs commands = From 148123de7aeb613151ed9cd49cadf3b088d5fdac Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 17:19:39 -0500 Subject: [PATCH 11/19] try to fix subprocess command --- hissw/environment.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index 44eda92..d5efe9d 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -4,6 +4,7 @@ import datetime import os import pathlib +import platform import stat import subprocess import tempfile @@ -188,12 +189,12 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) date_string = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') - # Get filenames + # Construct temporary filenames save_filename = tmpdir / f'idl_vars_{date_string}.sav' procedure_filename = tmpdir / f'idl_procedure_{date_string}.pro' command_filename = tmpdir / f'idl_script_{date_string}.pro' shell_filename = tmpdir / f'ssw_shell_{date_string}.sh' - # Render and save scripts + # Render and write scripts idl_script = self.custom_script(script, args) files = [ (procedure_filename, self.procedure_script(idl_script, save_vars, save_filename)), @@ -206,8 +207,10 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): f.write(filescript) # Execute os.chmod(shell_filename, mode=stat.S_IRWXU) + on_windows = platform.system().lower() == 'windows' cmd_output = subprocess.run( - [shell_filename], + shell_filename.name if on_windows else f'./{shell_filename.name}', + cwd=shell_filename.parent, shell=True, capture_output=True, text=True, From 130f9ee85464e6428fcb9eb0ce90ad9c8521b870 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 17:26:21 -0500 Subject: [PATCH 12/19] debugging ci --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a552d6..d6040cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: with: # NOTE: These are purposefully set to meaningless directories where there is no SSW # or IDL installation so that the relevant tests will be skipped. - posargs: '--ssw-home=/home/foo --idl-home=/home/bar' + posargs: '--ssw-home=/home/foo --idl-home=/home/bar --log-level=DEBUG' toxdeps: tox-pypi-filter envs: | - macos: py311 From 4b79ba021e2bf9aeabbdc43b036f99902c38fc74 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 17:53:24 -0500 Subject: [PATCH 13/19] more ci debugging --- hissw/environment.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index d5efe9d..eb69862 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -187,13 +187,13 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): # Expose the ssw_home variable in all scripts by default args.update({'ssw_home': self.ssw_home}) with tempfile.TemporaryDirectory() as tmpdir: - tmpdir = pathlib.Path(tmpdir) + tmpdir_path = pathlib.Path(tmpdir) date_string = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') # Construct temporary filenames - save_filename = tmpdir / f'idl_vars_{date_string}.sav' - procedure_filename = tmpdir / f'idl_procedure_{date_string}.pro' - command_filename = tmpdir / f'idl_script_{date_string}.pro' - shell_filename = tmpdir / f'ssw_shell_{date_string}.sh' + save_filename = tmpdir_path / f'idl_vars_{date_string}.sav' + procedure_filename = tmpdir_path / f'idl_procedure_{date_string}.pro' + command_filename = tmpdir_path / f'idl_script_{date_string}.pro' + shell_filename = tmpdir_path / f'ssw_shell_{date_string}.sh' # Render and write scripts idl_script = self.custom_script(script, args) files = [ @@ -206,20 +206,23 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): with open(filename, 'w') as f: f.write(filescript) # Execute - os.chmod(shell_filename, mode=stat.S_IRWXU) - on_windows = platform.system().lower() == 'windows' - cmd_output = subprocess.run( - shell_filename.name if on_windows else f'./{shell_filename.name}', - cwd=shell_filename.parent, - shell=True, - capture_output=True, - text=True, - ) - self._check_for_errors(cmd_output, raise_exceptions=raise_exceptions) + self._run_shell_script(shell_filename, raise_exceptions) results = readsav(save_filename) return results + def _run_shell_script(self, path, raise_exceptions): + os.chmod(path, mode=stat.S_IRWXU) + on_windows = platform.system().lower() == 'windows' + cmd_output = subprocess.run( + path.name if on_windows else f'./{path.name}', + cwd=path.parent, + shell=True, + capture_output=True, + text=True, + ) + self._check_for_errors(cmd_output, raise_exceptions=raise_exceptions) + def _check_for_errors(self, output, raise_exceptions=True): """ Check IDL output to try and decide if an error has occurred From 782766a3e1c4ebe9d957cdd1d809b5089894bcfb Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 17:56:45 -0500 Subject: [PATCH 14/19] more ci debugging --- hissw/environment.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index eb69862..ce4eb5c 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -4,7 +4,6 @@ import datetime import os import pathlib -import platform import stat import subprocess import tempfile @@ -213,10 +212,11 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): def _run_shell_script(self, path, raise_exceptions): os.chmod(path, mode=stat.S_IRWXU) - on_windows = platform.system().lower() == 'windows' + # on_windows = platform.system().lower() == 'windows' cmd_output = subprocess.run( - path.name if on_windows else f'./{path.name}', - cwd=path.parent, + # path.name if on_windows else f'./{path.name}', + # cwd=path.parent, + [str(path)], shell=True, capture_output=True, text=True, From 45ea5db21616ec15bbb4bc80dbbf0ad31976ea3f Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 18:04:39 -0500 Subject: [PATCH 15/19] more ci debugging --- hissw/environment.py | 18 ++++++++++-------- pyproject.toml | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index ce4eb5c..62b84be 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -4,6 +4,7 @@ import datetime import os import pathlib +import platform import stat import subprocess import tempfile @@ -212,14 +213,15 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): def _run_shell_script(self, path, raise_exceptions): os.chmod(path, mode=stat.S_IRWXU) - # on_windows = platform.system().lower() == 'windows' + on_windows = platform.system().lower() == 'windows' cmd_output = subprocess.run( - # path.name if on_windows else f'./{path.name}', - # cwd=path.parent, - [str(path)], + path.name if on_windows else f'./{path.name}', + cwd=path.parent, shell=True, - capture_output=True, - text=True, + # capture_output=True, + # text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) self._check_for_errors(cmd_output, raise_exceptions=raise_exceptions) @@ -227,8 +229,8 @@ def _check_for_errors(self, output, raise_exceptions=True): """ Check IDL output to try and decide if an error has occurred """ - stdout = output.stdout - stderr = output.stderr + stdout = f"{output.stdout.decode('utf-8')}" + stderr = f"{output.stderr.decode('utf-8')}" # NOTE: For some reason, not only errors are output to stderr so we # have to check it for certain keywords to see if an error occurred if raise_exceptions: diff --git a/pyproject.toml b/pyproject.toml index f539751..8f26a07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ extend-ignore = [ "PT007", # Parametrize should be lists of tuples # TODO! fix "PT011", # Too broad exception assert # TODO! fix "PT023", # Always use () on pytest decorators + "UP022", ] [tool.ruff.per-file-ignores] From 859b270364bcaa553f4358441d1e23ecbd82aebf Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 18:11:46 -0500 Subject: [PATCH 16/19] more ci debugging --- hissw/environment.py | 13 ++++++------- pyproject.toml | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index 62b84be..a840ea1 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -187,7 +187,8 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): # Expose the ssw_home variable in all scripts by default args.update({'ssw_home': self.ssw_home}) with tempfile.TemporaryDirectory() as tmpdir: - tmpdir_path = pathlib.Path(tmpdir) + tmpdir_path = pathlib.Path(tmpdir) / 'hissw_files' + tmpdir_path.mkdir(parents=True, exist_ok=True) date_string = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') # Construct temporary filenames save_filename = tmpdir_path / f'idl_vars_{date_string}.sav' @@ -218,10 +219,8 @@ def _run_shell_script(self, path, raise_exceptions): path.name if on_windows else f'./{path.name}', cwd=path.parent, shell=True, - # capture_output=True, - # text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, + text=True, ) self._check_for_errors(cmd_output, raise_exceptions=raise_exceptions) @@ -229,8 +228,8 @@ def _check_for_errors(self, output, raise_exceptions=True): """ Check IDL output to try and decide if an error has occurred """ - stdout = f"{output.stdout.decode('utf-8')}" - stderr = f"{output.stderr.decode('utf-8')}" + stdout = output.stdout + stderr = output.stderr # NOTE: For some reason, not only errors are output to stderr so we # have to check it for certain keywords to see if an error occurred if raise_exceptions: diff --git a/pyproject.toml b/pyproject.toml index 8f26a07..f539751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,6 @@ extend-ignore = [ "PT007", # Parametrize should be lists of tuples # TODO! fix "PT011", # Too broad exception assert # TODO! fix "PT023", # Always use () on pytest decorators - "UP022", ] [tool.ruff.per-file-ignores] From 0139fea9f79be4bfa88317af403c216c229cf7db Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Fri, 5 Jan 2024 18:17:13 -0500 Subject: [PATCH 17/19] endless ci debugging --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6040cd..45356ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: with: # NOTE: These are purposefully set to meaningless directories where there is no SSW # or IDL installation so that the relevant tests will be skipped. - posargs: '--ssw-home=/home/foo --idl-home=/home/bar --log-level=DEBUG' + posargs: '--ssw-home=/foo/bar --idl-home=/hello/world --log-level=DEBUG' toxdeps: tox-pypi-filter envs: | - macos: py311 From 653eaddf7d83ab245d7ad22799f8467e0721f35f Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Thu, 11 Jan 2024 16:42:46 -0500 Subject: [PATCH 18/19] fix pyproject.toml --- pyproject.toml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f539751..cb5684a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ "setuptools", - "setuptools_scm", + "setuptools_scm[toml]", "wheel", ] build-backend = 'setuptools.build_meta' @@ -9,13 +9,9 @@ build-backend = 'setuptools.build_meta' [project] name = "hissw" dynamic = ["version"] -requires-python = ">=3.9" -dependencies = [ - "jinja2", - "scipy", -] description = "Seamlessly integrate SSWIDL code into your Python workflow" readme = {file="README.md", content-type = "text/markdown"} +license = { file = "LICENSE.rst" } authors = [ {name="Will Barnes", email="will.t.barnes@gmail.com"}, {name="Bin Chen"}, @@ -35,6 +31,11 @@ keywords = [ "sswidl", "solarsoft", ] +requires-python = ">=3.9" +dependencies = [ + "jinja2", + "scipy", +] [project.urls] Homepage = "https://github.com/wtbarnes/hissw/" @@ -42,15 +43,14 @@ Documentation = "https://wtbarnes.github.io/hissw/" Source = "https://github.com/wtbarnes/hissw/" [project.optional-dependencies] -astropy = ["astropy"] -test = ["pytest", "astropy"] -docs = ["mkdocs", "mkdocs-material"] - -[tool.setuptools.packages.find] -where = ["hissw"] +all = ["hissw"] +astropy = ["hissw", "astropy"] +test = ["hissw[all]", "pytest", "astropy"] +docs = ["hissw[all]", "mkdocs", "mkdocs-material"] +dev = ["hissw[test,docs]"] -[tool.setuptools.package-data] -hissw = ["templates/*.pro", "templates/*.sh"] +[tool.setuptools] +packages = ["hissw"] [tool.setuptools_scm] From 0db96f12e0ecaa8455bc08e9f3474690df6d36f9 Mon Sep 17 00:00:00 2001 From: Will Barnes Date: Tue, 23 Jan 2024 09:35:22 -0500 Subject: [PATCH 19/19] us os.path.isfile to accomodate both script strings and paths --- hissw/environment.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hissw/environment.py b/hissw/environment.py index a840ea1..a8435df 100644 --- a/hissw/environment.py +++ b/hissw/environment.py @@ -117,7 +117,9 @@ def render_script(self, script, args): """ Render custom IDL scripts from templates and input arguments """ - if isinstance(script, (str, pathlib.Path)) and pathlib.Path(script).is_file(): + # NOTE: purposefully using os.path.isfile instead of pathlib.Path.is_file + # because the latter may throw an exception when the string is really long. + if isinstance(script, (str, pathlib.Path)) and os.path.isfile(script): with open(script) as f: script = f.read() if not isinstance(script, str): @@ -184,8 +186,6 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): """ save_vars = [] if save_vars is None else save_vars args = {} if args is None else args - # Expose the ssw_home variable in all scripts by default - args.update({'ssw_home': self.ssw_home}) with tempfile.TemporaryDirectory() as tmpdir: tmpdir_path = pathlib.Path(tmpdir) / 'hissw_files' tmpdir_path.mkdir(parents=True, exist_ok=True) @@ -196,7 +196,8 @@ def run(self, script, args=None, save_vars=None, raise_exceptions=True): command_filename = tmpdir_path / f'idl_script_{date_string}.pro' shell_filename = tmpdir_path / f'ssw_shell_{date_string}.sh' # Render and write scripts - idl_script = self.custom_script(script, args) + # NOTE: Expose the ssw_home variable in all scripts by default + idl_script = self.custom_script(script, {'ssw_home': self.ssw_home, **args}) files = [ (procedure_filename, self.procedure_script(idl_script, save_vars, save_filename)), (command_filename, self.command_script(procedure_filename)),