Merge branch 'develop'

.github/workflows/qa.yml

@@ -24,14 +24,12 @@ jobs:
     name: Code QA
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - run: pip install black flake8 isort
-    - run: black --version
-    - run: isort --version
-    - run: flake8 --version
-    - run: isort --check .
-    - run: black --check .
-    - run: flake8 .
+    - run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+    - uses: pre-commit/[email protected]
 
   short-tests:
     name: Short tests
@@ -44,4 +42,4 @@ jobs:
     - name: Install climetlab
       run: pip install -e .
     - run: pip install pytest
-    - run: pytest -vv -E short
+    - run: pytest -vv -E short

.github/workflows/test-and-release.yml

@@ -35,34 +35,12 @@ jobs:
     name: Code QA
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - run: pip install black flake8 isort
-    - run: black --version
-    - run: isort --version
-    - run: flake8 --version
-    - run: isort --check .
-    - run: black --check .
-    - run: flake8 .
-
-  # test_build_doc:
-  #   name: Test building documentation
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #   - uses: actions/checkout@v3
-
-  #   - name: Install packages needed to build the documentation
-  #     run: |
-  #       pip install sphinx
-  #       pip install -r docs/requirements.txt
-  #       sudo apt-get install pandoc
-  #       pip freeze
-
-  #   - name: Documentation
-  #     run: |
-  #       make clean
-  #       make html
-  #       # make linkcheck (don't run yet)
-  #     working-directory: docs
+    - run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+    - uses: pre-commit/[email protected]
 
   download-test-data:
     name: Download test data
@@ -71,7 +49,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: 3.x
 
     - name: Tokens
       shell: python
@@ -114,7 +92,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Tokens 
+    - name: Tokens
       shell: python
       env:
         ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
@@ -151,7 +129,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Tokens 
+    - name: Tokens
       shell: python
       env:
         ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
@@ -172,19 +150,16 @@ jobs:
         pip install pytest
         pip freeze
     - name: Tests notebooks
-      run: pytest tests/documentation/test_notebooks.py    
+      run: pytest tests/documentation/test_notebooks.py
 
   short-tests:
-    # if: (github.event_name == 'release' && github.event.action == 'created') || github.ref == 'refs/heads/main'
-    # if: github.event_name == 'release' && github.event.action == 'created'
-    # if: github.ref == 'refs/heads/main'
+
     needs: [download-test-data]
     strategy:
       matrix:
         platform: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.8", "3.9", "3.10"]
-        # platform: ["ubuntu-latest"]
-        # python-version: ["3.10"]
+        python-version: ["3.9", "3.10"]
+
     name: Short tests Python ${{ matrix.python-version }} ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
     steps:
@@ -199,7 +174,8 @@ jobs:
         name: test-data
     - run: tar xvf test-data.tar
     - run: cat test-data/dataA.txt
-    - name: Tokens 
+
+    - name: Tokens
       # (this should be removed when tests are refactored)
       shell: python
       env:
@@ -213,8 +189,12 @@ jobs:
         if os.environ[n]:
             with open(m, "w") as f:
                 print(os.environ[n], file=f)
+
+    - run: pip install --upgrade pip
+
     - name: Install climetlab
       run: pip install -e .
+
     - run: climetlab versions
     - name: Install test tools
       run: |
@@ -231,51 +211,29 @@ jobs:
         # notebooks need more dependencies
         # pytest --durations=0 -E release -k 'not test_notebooks'
         pytest --durations=10 -vv -E short
-   #  - name: Install climetlab full
-   #    run: |
-   #      pip install .[interactive,tensorflow,zarr]
-   #      pip freeze
-   #  - name: Tests with dependencies
-   #    run: |
-   #      pytest --durations=10 -vv -E short
 
-  check-version-tag:
-    if: github.event_name == 'release' && github.event.action == 'created'
-    name: Check versions and tags
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.8"
-    - name: Check that tag version matches package version
-      run: |
-        tag=${GITHUB_REF#refs/tags/}
-        version=$(python setup.py --version)
-        echo 'tag='$tag
-        echo "version file="$version
-        test "$tag" == "$version"
 
   deploy:
     name: Upload to Pypi and release
-    needs: [check-version-tag, short-tests, quality, tests-with-external-download, tests-notebooks]
-    # needs: [check-version-tag, short-tests, quality, test_build_doc, tests-with-external-download, tests-notebooks]
+    needs: [short-tests, quality, tests-with-external-download, tests-notebooks]
     runs-on: ubuntu-latest
+
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.9"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build pip package
-      run: python setup.py sdist
-    - name: Publish
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
-      run: twine upload --verbose dist/*
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          python -m build
+          twine upload dist/*

.gitignore

@@ -171,3 +171,4 @@ tempCodeRunnerFile.py
 *.dot
 *.zarr/
 testdata/
+_version.py

.pre-commit-config.yaml

@@ -1,22 +1,40 @@
-exclude: 'experiments'
 repos:
--   repo: https://github.com/ambv/black
-    rev: 24.1.1
-    hooks:
-      - id: black
-        language_version: python3.10
--   repo: https://github.com/pycqa/isort
-    rev: 5.13.0
-    hooks:
-      - id: isort
-        name: isort (python)
-      - id: isort
-        name: isort (cython)
-        types: [cython]
-      - id: isort
-        name: isort (pyi)
-        types: [pyi]
--   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
-    hooks:
-    -   id: flake8
+
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
+  - id: end-of-file-fixer # Ensure files end in a newline
+  - id: trailing-whitespace # Trailing whitespace checker
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-added-large-files # Check for large files added to git
+  - id: check-merge-conflict # Check for files that contain merge conflict
+
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.1.1
+  hooks:
+    - id: black
+      args: [--line-length=120]
+
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
+
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.3.0
+  hooks:
+  - id: ruff
+    args:
+    - --line-length=120
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview

README.md

@@ -15,7 +15,5 @@ The documentation can be found at <https://climetlab.readthedocs.io/>.
 See https://climetlab.readthedocs.io/en/latest/guide/pluginlist.html.
 
 ### License
-[Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities 
+[Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities
 granted to it by virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
-
-

climetlab-and-ecmwflibs-install.md

@@ -1,9 +1,9 @@
 # How to run climetlab without ecmwflibs (using conda)
 
-This workaround should address some of the issues related to ecmwflibs version not found or not installing correctly occuring during the installation of the climetlab package. 
+This workaround should address some of the issues related to ecmwflibs version not found or not installing correctly occuring during the installation of the climetlab package.
 This is how to install eccodes manually and use an environment variable to avoid installing ecmwflibs.
 
-``` 
+```
 # install eccodes manually
 conda install -c conda-forge eccodes
 pip install eccodes
@@ -15,7 +15,7 @@ CLIMETLAB_DO_NOT_INSTALL_ECMWFLIBS=1 pip install climetlab
 # Limitations
 
 Obviously installing climetlab without one of its dependencies has some impact on its features.
-And installing eccodes is not enough to provide them all : more packages are bundled in ecmwflibs (other than eccodes). 
+And installing eccodes is not enough to provide them all : more packages are bundled in ecmwflibs (other than eccodes).
 Depending on the required functinalities, there may be other missing packages to install manually.
 
 |    Feature    | default (ecmwflibs) | With workaround above |
@@ -29,7 +29,7 @@ All CliMetLab features related to **reading (or writing) GRIB** data be supporte
 # Debugging
 
 ## Check eccodes installation
-``` 
+```
 import eccodes
 print(eccodes.__file__)
 ```

dev/dev.ipynb

@@ -94,4 +94,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

dev/in-progress.ipynb

@@ -1162,4 +1162,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

dev/tf.py

@@ -1,17 +1,11 @@
-import keras.layers as layers
-import numpy as np
 import tensorflow as tf
-import tensorflow.keras as keras
-from tensorflow.keras.callbacks import EarlyStopping, TensorBoard
-from tensorflow.keras.layers import (
-    Conv2D,
-    Dense,
-    Dropout,
-    Flatten,
-    Input,
-    Lambda,
-    Reshape,
-)
+from tensorflow.keras.callbacks import EarlyStopping
+from tensorflow.keras.layers import Conv2D
+from tensorflow.keras.layers import Dense
+from tensorflow.keras.layers import Dropout
+from tensorflow.keras.layers import Flatten
+from tensorflow.keras.layers import Input
+from tensorflow.keras.layers import Reshape
 from tensorflow.keras.models import Sequential
 
 import climetlab as cml
@@ -46,9 +40,7 @@ def __init__(self, *args, **kwargs):
     def call(self, inputs):
         w, h = self.kernel_size
         inputs = tf.concat([inputs, inputs[:, :, :w, :]], axis=2)
-        inputs = tf.pad(
-            inputs, [[0, 0], [h // 2, h // 2], [0, 0], [0, 0]], constant_values=0
-        )
+        inputs = tf.pad(inputs, [[0, 0], [h // 2, h // 2], [0, 0], [0, 0]], constant_values=0)
         return super().call(inputs)