From 2873576a2d86afd5b04db33760b8eaffc346088a Mon Sep 17 00:00:00 2001 From: Mirek Simek Date: Tue, 31 Oct 2023 20:44:21 +0100 Subject: [PATCH] Upgraded imports, preparing for oarepo 12 (#3) * Upgraded imports, preparing for oarepo 12 * creating venv --- .github/workflows/build.yaml | 78 +++++++++++++++ .github/workflows/main.yml | 96 ------------------- .github/workflows/manual.yaml | 15 +++ .github/workflows/push.yaml | 36 +++++++ .gitignore | 4 +- .../builders/tests/conftest.py | 2 +- .../draft_file_model/files_field.py | 20 ++-- .../components/draft_file_profile.py | 2 +- .../components/drafts_files_record.py | 9 +- run-tests.sh | 35 +++++++ run_tests.sh | 20 ---- setup.cfg | 2 +- 12 files changed, 181 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/manual.yaml create mode 100644 .github/workflows/push.yaml create mode 100755 run-tests.sh delete mode 100755 run_tests.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..ecb855c --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,78 @@ +name: Build and test + +on: + workflow_call: + inputs: + oarepo: + description: OARepo version (11, 12, ...) + required: true + default: 11 + type: string + +env: + OAREPO_VERSION: ${{ github.event.inputs.oarepo }} + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.9", "3.10" ] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Configure sysctl limits + run: | + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + + - name: Runs Opensearch + uses: ankane/setup-opensearch@v1 + with: + plugins: analysis-icu + + - name: Start Redis + uses: supercharge/redis-github-action@1.7.0 + with: + redis-version: ${{ matrix.redis-version }} + + - name: Run tests + run: | + ./run-tests.sh + + - name: Build package to publish + run: | + .venv-builder/bin/python setup.py sdist bdist_wheel + + - name: Freeze packages + run: | + .venv-builder/bin/pip freeze > requirements.txt + .venv-tests/bin/pip freeze >>requirements.txt + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + - name: Archive production artifacts + uses: actions/upload-artifact@v3 + with: + name: requirements.txt + path: requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 2e29bd2..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Test python - -on: [ push ] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ "3.9", "3.10" ] - search-service: [opensearch2] - include: - - search-service: opensearch2 - SEARCH_EXTRAS: "opensearch2" - env: - SEARCH: ${{ matrix.search-service }} - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v3 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - name: Configure sysctl limits - run: | - sudo swapoff -a - sudo sysctl -w vm.swappiness=1 - sudo sysctl -w fs.file-max=262144 - sudo sysctl -w vm.max_map_count=262144 - - name: Runs Opensearch - uses: ankane/setup-opensearch@v1 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip wheel setuptools - pip install -e . - pip list - - name: Test with pytest - run: | - sh run_tests.sh - # pytest tests - # - # - name: Coveralls - # uses: AndreMiras/coveralls-python-action@develop - # with: - # parallel: true - # flag-name: Test with pytest - - # coveralls_finish: - # needs: build - # runs-on: ubuntu-latest - # steps: - # - name: Coveralls Finished - # uses: AndreMiras/coveralls-python-action@develop - # with: - # parallel-finished: true - - - name: Build package to publish - run: | - python setup.py sdist bdist_wheel - - name: Archive production artifacts - uses: actions/upload-artifact@v3 - with: - name: dist - path: dist - - deploy: - runs-on: ubuntu-latest - needs: build - steps: - - name: Use built artifacts - uses: actions/download-artifact@v3 - with: - name: dist - path: dist - - - name: List files - run: | - ls -la - ls -la dist - - name: Publish package - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - skip_existing: true - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/manual.yaml b/.github/workflows/manual.yaml new file mode 100644 index 0000000..e05df2c --- /dev/null +++ b/.github/workflows/manual.yaml @@ -0,0 +1,15 @@ +name: Dispatch + +on: + workflow_dispatch: + inputs: + oarepo: + description: OARepo version (11, 12, ...) + required: true + default: 11 + +jobs: + build: + uses: ./.github/workflows/build.yaml + with: + oarepo: ${{ github.event.inputs.oarepo }} diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 0000000..fad9a87 --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,36 @@ +name: Build, test and publish + +on: push + +permissions: + id-token: write + contents: read + +jobs: + build: + uses: ./.github/workflows/build.yaml + with: + oarepo: 11 + + publish: + runs-on: ubuntu-latest + needs: build + steps: + - name: Use built artifacts + uses: actions/download-artifact@v3 + with: + name: dist + path: dist + + - name: List files + run: | + ls -la + ls -la dist + + - name: Publish package + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip_existing: true + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.gitignore b/.gitignore index 35f32fd..1bc1388 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ __pycache__/ .Python env/ venv/ -.venv/ +.venv*/ .model_venv/ build/ develop-eggs/ @@ -91,3 +91,5 @@ dist/ .model_venv /tests/test /thesis/ + +build-tests/thesis \ No newline at end of file diff --git a/oarepo_model_builder_drafts_files/builders/tests/conftest.py b/oarepo_model_builder_drafts_files/builders/tests/conftest.py index a78a634..61c4ff4 100644 --- a/oarepo_model_builder_drafts_files/builders/tests/conftest.py +++ b/oarepo_model_builder_drafts_files/builders/tests/conftest.py @@ -14,4 +14,4 @@ def finish(self, **extra_kwargs): ) def _get_output_module(self): - return f'{self.current_model.definition["tests"]["module"]}.conftest' \ No newline at end of file + return f'{self.current_model.definition["tests"]["module"]}.conftest' diff --git a/oarepo_model_builder_drafts_files/datatypes/components/draft_file_model/files_field.py b/oarepo_model_builder_drafts_files/datatypes/components/draft_file_model/files_field.py index 4c65fa1..35eab94 100644 --- a/oarepo_model_builder_drafts_files/datatypes/components/draft_file_model/files_field.py +++ b/oarepo_model_builder_drafts_files/datatypes/components/draft_file_model/files_field.py @@ -15,18 +15,16 @@ def before_model_prepare(self, datatype, *, context, **kwargs): return files_field = set_default(datatype, "files-field", {}) if datatype.root.profile == "draft_files": - record_class = datatype.definition["record"]["class"] - files_field.setdefault("file-class", base_name(record_class)) + files_field.setdefault("file-class", datatype.definition["record"]["class"]) files_field.setdefault("field-args", ["store=False", "delete=False"]) - files_field.setdefault( - "imports", - [ - { - "import": "invenio_records_resources.records.systemfields.FilesField" - }, - {"import": record_class}, - ], - ) + # files_field.setdefault( + # "imports", + # [ + # { + # "import": "invenio_records_resources.records.systemfields.FilesField" + # }, + # ], + # ) if datatype.root.profile == "files": files_field.setdefault( diff --git a/oarepo_model_builder_drafts_files/datatypes/components/draft_file_profile.py b/oarepo_model_builder_drafts_files/datatypes/components/draft_file_profile.py index 68e877e..5d094d9 100644 --- a/oarepo_model_builder_drafts_files/datatypes/components/draft_file_profile.py +++ b/oarepo_model_builder_drafts_files/datatypes/components/draft_file_profile.py @@ -2,10 +2,10 @@ from oarepo_model_builder.datatypes import ( DataType, DataTypeComponent, - Import, ModelDataType, Section, ) +from oarepo_model_builder.utils.python_name import Import from oarepo_model_builder.datatypes.components import DefaultsModelComponent from oarepo_model_builder.datatypes.components.model.utils import set_default from oarepo_model_builder.datatypes.model import Link diff --git a/oarepo_model_builder_drafts_files/datatypes/components/drafts_files_record.py b/oarepo_model_builder_drafts_files/datatypes/components/drafts_files_record.py index 025fe91..993b9f9 100644 --- a/oarepo_model_builder_drafts_files/datatypes/components/drafts_files_record.py +++ b/oarepo_model_builder_drafts_files/datatypes/components/drafts_files_record.py @@ -17,13 +17,8 @@ def before_model_prepare(self, datatype, *, context, **kwargs): append_array( datatype, "service-config", - "imports", - { - "import": "invenio_drafts_resources.services.records.components.DraftFilesComponent" - }, - ) - append_array( - datatype, "service-config", "components", "DraftFilesComponent" + "components", + "invenio_drafts_resources.services.records.components.DraftFilesComponent", ) service = set_default(datatype, "service", {}) service.setdefault( diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 0000000..f923d56 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e + +OAREPO_VERSION=${OAREPO_VERSION:-11} +OAREPO_VERSION_MAX=$((OAREPO_VERSION+1)) + +BUILDER_VENV=".venv-builder" +if test -d $BUILDER_VENV ; then + rm -rf $BUILDER_VENV +fi + +python3 -m venv $BUILDER_VENV +. $BUILDER_VENV/bin/activate +pip install -U setuptools pip wheel +pip install -e . + + +MODEL="thesis" +VENV=".venv-tests" + +if test -d ./build-tests/$MODEL; then + rm -rf ./build-tests/$MODEL +fi + +if test -d ./$VENV; then + rm -rf ./$VENV +fi + +oarepo-compile-model ./build-tests/$MODEL.yaml --output-directory ./build-tests/$MODEL -vvv +python3 -m venv $VENV +. $VENV/bin/activate +pip install -U setuptools pip wheel +pip install "oarepo>=$OAREPO_VERSION,<$OAREPO_VERSION_MAX" +pip install "./build-tests/${MODEL}[tests]" +pytest build-tests/$MODEL/tests \ No newline at end of file diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index dd90d7e..0000000 --- a/run_tests.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e - -MODEL="thesis" -VENV=".model_venv" - -if test -d ./build-tests/$MODEL; then - rm -rf ./build-tests/$MODEL -fi - -if test -d ./$VENV; then - rm -rf ./$VENV -fi - -oarepo-compile-model ./build-tests/$MODEL.yaml --output-directory ./build-tests/$MODEL -vvv -python3 -m venv $VENV -. $VENV/bin/activate -pip install -U setuptools pip wheel -pip install "./build-tests/$MODEL[tests]" -pytest build-tests/$MODEL/tests \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 75b258c..6956b4c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = oarepo-model-builder-drafts-files -version = 4.0.4 +version = 4.0.5 description = authors = Ronald Krist readme = README.md