From 6748b7b3c18265c7baab89411acc1c3dd6b2f61b Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Fri, 8 Dec 2023 17:50:16 +0100 Subject: [PATCH 01/15] define and use pinned requirements --- .docker/run.sh | 2 +- Dockerfile | 13 +++++++-- dev-requirements.in | 9 ++++++ dev-requirements.txt | 65 ++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 7 +++++ requirements.txt | 39 ++++++++++++++++++++++++++ 6 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 dev-requirements.in create mode 100644 dev-requirements.txt create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/.docker/run.sh b/.docker/run.sh index 6d483f1..c648101 100644 --- a/.docker/run.sh +++ b/.docker/run.sh @@ -4,6 +4,6 @@ INIPASTE=${INIPASTE:-production.ini} cd $APP_ROOT -. /opt/apps/convertit/bin/activate +. /opt/venv/bin/activate gunicorn --workers=1 --paste=$INIPASTE --bind=0.0.0.0:6543 diff --git a/Dockerfile b/Dockerfile index 039528b..6fbdaf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,12 +18,19 @@ FROM base as build RUN apt-get update && apt-get install -y -qq build-essential python3-venv python3-dev && \ apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/* -ADD . /opt/apps/convertit +COPY requirements.txt /requirements.txt -RUN python3 -m venv . && ./bin/pip install --no-cache-dir pip setuptools wheel -U -RUN ./bin/python setup.py install +RUN python3 -m venv /opt/venv && /opt/venv/bin/pip install --no-cache-dir pip setuptools wheel -U && /opt/venv/bin/pip install --no-cache-dir -U -r /requirements.txt + +FROM build as dev + +COPY dev-requirements.txt /dev-requirements.txt + +RUN /opt/venv/bin/pip install --no-cache-dir -U -r /dev-requirements.txt FROM base as prod COPY --from=build /opt/apps/convertit /opt/apps/convertit +COPY convertit /opt/apps/convertit/convertit +COPY setup.py /opt/apps/convertit/setup.py ADD .docker/run.sh /usr/local/bin/run diff --git a/dev-requirements.in b/dev-requirements.in new file mode 100644 index 0000000..d2843a7 --- /dev/null +++ b/dev-requirements.in @@ -0,0 +1,9 @@ +-c requirements.txt + +webtest +mock +coverage +freezegun +flake8 +isort +pip-tools \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..b6d1134 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,65 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile dev-requirements.in +# +beautifulsoup4==4.12.2 + # via webtest +build==1.0.3 + # via pip-tools +click==8.1.7 + # via pip-tools +coverage==7.3.2 + # via -r dev-requirements.in +flake8==6.1.0 + # via -r dev-requirements.in +freezegun==1.3.1 + # via -r dev-requirements.in +importlib-metadata==7.0.0 + # via build +isort==5.12.0 + # via -r dev-requirements.in +mccabe==0.7.0 + # via flake8 +mock==5.1.0 + # via -r dev-requirements.in +packaging==23.2 + # via + # -c requirements.txt + # build +pip-tools==7.3.0 + # via -r dev-requirements.in +pycodestyle==2.11.1 + # via flake8 +pyflakes==3.1.0 + # via flake8 +pyproject-hooks==1.0.0 + # via build +python-dateutil==2.8.2 + # via freezegun +six==1.16.0 + # via python-dateutil +soupsieve==2.5 + # via beautifulsoup4 +tomli==2.0.1 + # via + # build + # pip-tools + # pyproject-hooks +waitress==2.1.2 + # via webtest +webob==1.8.7 + # via + # -c requirements.txt + # webtest +webtest==3.0.0 + # via -r dev-requirements.in +wheel==0.42.0 + # via pip-tools +zipp==3.17.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..39eccb6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: "3.8" +services: + web: + build: + target: dev + volumes: + - .:/opt/apps/convertit \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4bec064 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,39 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile +# +gunicorn==21.2.0 + # via convertit (setup.py) +hupper==1.12 + # via pyramid +packaging==23.2 + # via gunicorn +pastedeploy==3.1.0 + # via plaster-pastedeploy +pillow==10.1.0 + # via convertit (setup.py) +plaster==1.1.2 + # via + # plaster-pastedeploy + # pyramid +plaster-pastedeploy==1.0.1 + # via pyramid +pyramid==1.10.5 + # via convertit (setup.py) +python-magic==0.4.27 + # via convertit (setup.py) +translationstring==1.4 + # via pyramid +venusian==3.1.0 + # via pyramid +webob==1.8.7 + # via pyramid +zope-deprecation==5.0 + # via pyramid +zope-interface==6.1 + # via pyramid + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From c18a90c2a02c3dc6e258cd755e1efd0d034e9626 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 12:27:48 +0100 Subject: [PATCH 02/15] define and use pinned requirements --- .docker/Dockerfile.debian.builder | 4 ++-- debian/rules | 1 + requirements.in | 4 ++++ requirements.txt | 8 ++++---- setup.py | 24 ++++++++---------------- 5 files changed, 19 insertions(+), 22 deletions(-) create mode 100644 requirements.in diff --git a/.docker/Dockerfile.debian.builder b/.docker/Dockerfile.debian.builder index 249500e..f414c8d 100644 --- a/.docker/Dockerfile.debian.builder +++ b/.docker/Dockerfile.debian.builder @@ -28,7 +28,7 @@ COPY . ./ WORKDIR /dpkg-build RUN sed -i -re "1s/..UNRELEASED/.ubuntu$(lsb_release -rs)) $(lsb_release -cs)/" debian/changelog \ - && chmod a-x debian/convertit.* \ - && dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \ + && chmod a-x debian/convertit.* +RUN dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /convertit[-_]* /dpkg \ && dpkg-deb -I /dpkg/convertit*.deb WORKDIR /dpkg diff --git a/debian/rules b/debian/rules index acc21a3..1eda09f 100755 --- a/debian/rules +++ b/debian/rules @@ -10,6 +10,7 @@ override_dh_virtualenv: --python /usr/bin/python3 \ --extra-pip-arg --no-cache-dir \ --builtin-venv + --preinstall wheel mkdir -p debian/convertit/usr/bin mkdir -p debian/convertit/opt/convertit cp production.ini debian/convertit/opt/convertit/convertit.ini diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..5bd04be --- /dev/null +++ b/requirements.in @@ -0,0 +1,4 @@ +pyramid<=1.10.5 +gunicorn +python-magic +Pillow \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4bec064..03926c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ # pip-compile # gunicorn==21.2.0 - # via convertit (setup.py) + # via -r requirements.in hupper==1.12 # via pyramid packaging==23.2 @@ -13,7 +13,7 @@ packaging==23.2 pastedeploy==3.1.0 # via plaster-pastedeploy pillow==10.1.0 - # via convertit (setup.py) + # via -r requirements.in plaster==1.1.2 # via # plaster-pastedeploy @@ -21,9 +21,9 @@ plaster==1.1.2 plaster-pastedeploy==1.0.1 # via pyramid pyramid==1.10.5 - # via convertit (setup.py) + # via -r requirements.in python-magic==0.4.27 - # via convertit (setup.py) + # via -r requirements.in translationstring==1.4 # via pyramid venusian==3.1.0 diff --git a/setup.py b/setup.py index edfd39a..175c94b 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,18 @@ import os +from pip.download import PipSession +from pip.req import parse_requirements from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst'), encoding='utf-8').read() CHANGES = open(os.path.join(here, 'CHANGES.rst'), encoding='utf-8').read() -requires = [ - 'pyramid<=1.10.5', - 'gunicorn', - 'python-magic', - 'Pillow', -] +install_reqs = parse_requirements('./requirements.txt', session=PipSession()) -test_requires = requires + [ - 'webtest', - 'mock', - 'coverage', - 'freezegun', - 'flake8', - 'isort' -] +reqs = [str(ir.req) for ir in install_reqs] + +test_reqs = reqs + [str(ir.req) for ir in parse_requirements('./dev-requirements.txt', session=PipSession())] setup(name='convertit', @@ -42,10 +34,10 @@ packages=find_packages(), include_package_data=True, zip_safe=False, - install_requires=requires, + #install_requires=reqs, tests_require=test_requires, extras_require={ - 'test': test_requires, + 'test': test_reqs, }, test_suite="convertit", entry_points="""\ From 4b2549ad0d345ec7b5469ead2b5603474a255a15 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 12:48:06 +0100 Subject: [PATCH 03/15] fix debian --- debian/rules | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/debian/rules b/debian/rules index 1eda09f..5297dbf 100755 --- a/debian/rules +++ b/debian/rules @@ -9,8 +9,10 @@ override_dh_virtualenv: dh_virtualenv \ --python /usr/bin/python3 \ --extra-pip-arg --no-cache-dir \ - --builtin-venv - --preinstall wheel + --upgrade-pip \ + --preinstall wheel \ + --preinstall setuptools \ + --builtin-venv \ mkdir -p debian/convertit/usr/bin mkdir -p debian/convertit/opt/convertit cp production.ini debian/convertit/opt/convertit/convertit.ini @@ -21,5 +23,4 @@ override_dh_strip: dh_strip --no-automatic-dbgsym override_dh_shlibdeps: - dh_shlibdeps \ - -X site-packages/Pillow.libs \ No newline at end of file + dh_shlibdeps --exclude=/site-packages/ \ No newline at end of file From 5f27fa57aa3995cbd1bdbf39fdea5de580335035 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 12:52:08 +0100 Subject: [PATCH 04/15] update CI --- .github/dependabot.yml | 12 ++++++++++++ .github/workflows/ci.yml | 30 +++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1b18eec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + open-pull-requests-limit: 20 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dabaea6..646c99a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,17 +56,20 @@ jobs: continue-on-error: true strategy: matrix: - version: [22.04, 20.04, docker] + version: ['24.04', '22.04', '20.04', docker] include: - - version: 22.04 + - version: '24.04' + distro: noble + - version: '22.04' distro: jammy - - version: 20.04 + - version: '20.04' distro: focal - version: docker distro: docker image + name: Build ${{ matrix.distro }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Prepare debian versioning if: ${{ matrix.version != 'docker' }} run: | @@ -104,16 +107,18 @@ jobs: image: "convertit:latest" e2e: - continue-on-error: true strategy: matrix: - distro: [ jammy, focal, docker ] + distro: [ noble, jammy, focal, docker ] include: + - distro: noble + version: '24.04' + - distro: jammy - version: 22.04 + version: '22.04' - distro: focal - version: 20.04 + version: '20.04' - distro: docker version: latest @@ -195,13 +200,20 @@ jobs: file -b -f ./myfile.pdf; exit 1; fi + deploy: name: Publish (on release only) runs-on: ubuntu-latest needs: [ flake8, isort, e2e ] if: ${{ github.event_name == 'release' && github.event.action == 'created' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - name: Download 24.04 debian artifact + uses: actions/download-artifact@v3 + with: + name: debian-noble + - name: Download 22.04 debian artifact uses: actions/download-artifact@v3 with: From e292c18b130264c7beb05c501366ede4cac0e50b Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 12:53:30 +0100 Subject: [PATCH 05/15] update CI --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 646c99a..8299d35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: - name: Archive package artifact if: ${{ matrix.version != 'docker' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: debian-${{ matrix.distro }} path: | @@ -127,7 +127,7 @@ jobs: runs-on: ubuntu-${{ matrix.version }} needs: [ build ] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ matrix.distro != 'docker' }} with: name: debian-${{ matrix.distro }} @@ -210,17 +210,17 @@ jobs: - uses: actions/checkout@v4 - name: Download 24.04 debian artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: debian-noble - name: Download 22.04 debian artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: debian-jammy - name: Download 20.04 debian artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: debian-focal From 0cff1d6fc63b50f03fbea2ade45c164683dd4fae Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 13:04:28 +0100 Subject: [PATCH 06/15] fix example files --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8299d35..9ab630f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,7 @@ jobs: - name: Test ODT / PDF conversion run: | - wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text" + wget -O myfile.pdf "http://localhost:6543/?url=https://file-examples.com/wp-content/storage/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text" if file -b --mime-type ./myfile.pdf |grep -q 'application/pdf'; then echo "File is PDF"; @@ -164,7 +164,7 @@ jobs: - name: Test ODT / DOC conversion run: | - wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/febf69dcf3656dfd992b0fa/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword" + wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/wp-content/storage/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword" if file -b --mime-type ./myfile.doc |grep -q 'application/msword'; then echo "File is DOC"; From bc14901972f0fd56dbf2136418185d69f560fd42 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 14:55:34 +0100 Subject: [PATCH 07/15] fix example files --- .docker/run.sh | 2 +- .github/workflows/ci.yml | 1 + Dockerfile | 19 ++++++++++++------- requirements.in | 1 - requirements.txt | 3 ++- setup.py | 17 +---------------- 6 files changed, 17 insertions(+), 26 deletions(-) diff --git a/.docker/run.sh b/.docker/run.sh index c648101..bdd782e 100644 --- a/.docker/run.sh +++ b/.docker/run.sh @@ -6,4 +6,4 @@ cd $APP_ROOT . /opt/venv/bin/activate -gunicorn --workers=1 --paste=$INIPASTE --bind=0.0.0.0:6543 +pserve $INIPASTE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ab630f..7dc4b72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,7 @@ jobs: if: ${{ matrix.distro == 'docker' }} run: | docker run -p 6543:6543 -d convertit:latest + s - name: Test ODT / PDF conversion run: | diff --git a/Dockerfile b/Dockerfile index 6fbdaf5..a500d95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ ARG DISTRO=jammy -FROM ubuntu:${DISTRO} as base -MAINTAINER Makina Corpus "contact@makina-corpus.com" +FROM ubuntu:${DISTRO} AS base +LABEL org.opencontainers.image.authors="Makina Corpus " + RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre libreoffice-java-common inkscape python3-magic && \ apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/* @@ -9,12 +10,12 @@ RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre lib WORKDIR /opt/apps/convertit -ADD .docker/run.sh /usr/local/bin/run +COPY .docker/run.sh /usr/local/bin/run EXPOSE 6543 CMD ["/bin/sh", "-e", "/usr/local/bin/run"] -FROM base as build +FROM base AS build RUN apt-get update && apt-get install -y -qq build-essential python3-venv python3-dev && \ apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/* @@ -22,15 +23,19 @@ COPY requirements.txt /requirements.txt RUN python3 -m venv /opt/venv && /opt/venv/bin/pip install --no-cache-dir pip setuptools wheel -U && /opt/venv/bin/pip install --no-cache-dir -U -r /requirements.txt -FROM build as dev + +FROM build AS dev COPY dev-requirements.txt /dev-requirements.txt RUN /opt/venv/bin/pip install --no-cache-dir -U -r /dev-requirements.txt -FROM base as prod +FROM base AS prod COPY --from=build /opt/apps/convertit /opt/apps/convertit +COPY --from=build /opt/venv /opt/venv COPY convertit /opt/apps/convertit/convertit COPY setup.py /opt/apps/convertit/setup.py -ADD .docker/run.sh /usr/local/bin/run +COPY README.rst /opt/apps/convertit/README.rst +RUN /opt/venv/bin/pip install . +COPY production.ini /opt/apps/convertit/production.ini diff --git a/requirements.in b/requirements.in index 5bd04be..34b1528 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,3 @@ pyramid<=1.10.5 -gunicorn python-magic Pillow \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 03926c5..fecd6bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,11 @@ # # pip-compile # -gunicorn==21.2.0 +gunicorn==23.0.0 # via -r requirements.in hupper==1.12 # via pyramid +importlib-metadata packaging==23.2 # via gunicorn pastedeploy==3.1.0 diff --git a/setup.py b/setup.py index 175c94b..3e1fad2 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,14 @@ import os -from pip.download import PipSession -from pip.req import parse_requirements from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst'), encoding='utf-8').read() -CHANGES = open(os.path.join(here, 'CHANGES.rst'), encoding='utf-8').read() - -install_reqs = parse_requirements('./requirements.txt', session=PipSession()) - -reqs = [str(ir.req) for ir in install_reqs] - -test_reqs = reqs + [str(ir.req) for ir in parse_requirements('./dev-requirements.txt', session=PipSession())] - setup(name='convertit', version='2.2.6.dev0', description='A file conversion Web API in Pyramid', - long_description=README + '\n\n' + CHANGES, + long_description=README, license='AGPLV3', classifiers=[ "License :: OSI Approved :: GNU Affero General Public License v3", @@ -34,11 +24,6 @@ packages=find_packages(), include_package_data=True, zip_safe=False, - #install_requires=reqs, - tests_require=test_requires, - extras_require={ - 'test': test_reqs, - }, test_suite="convertit", entry_points="""\ [paste.app_factory] From 8d677676891e4958855bf163aca1a42423227ba0 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 15:03:40 +0100 Subject: [PATCH 08/15] fix example files --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dc4b72..9ab630f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,7 +148,6 @@ jobs: if: ${{ matrix.distro == 'docker' }} run: | docker run -p 6543:6543 -d convertit:latest - s - name: Test ODT / PDF conversion run: | From cdad832d992c24e41e478e77d5b694b4a4972d2c Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Mon, 18 Nov 2024 15:11:41 +0100 Subject: [PATCH 09/15] fix example files --- production.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production.ini b/production.ini index f0a9eee..8ddac84 100644 --- a/production.ini +++ b/production.ini @@ -35,7 +35,7 @@ convertit.converters = [server:main] use = egg:gunicorn#main -host = localhost +host = 0.0.0.0 port = 6543 # Begin logging configuration From c1624b33fe7515c44be9ed511f7725925eec02d7 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 19 Nov 2024 08:58:35 +0100 Subject: [PATCH 10/15] fix example files --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ab630f..99ee185 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: - name: Upload image if: ${{ matrix.version == 'docker' }} - uses: ishworkh/docker-image-artifact-upload@v1 + uses: ishworkh/container-image-artifact-upload@v2.0.0 with: image: "convertit:latest" @@ -134,7 +134,7 @@ jobs: - name: Download image if: ${{ matrix.distro == 'docker' }} - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "convertit:latest" @@ -164,7 +164,7 @@ jobs: - name: Test ODT / DOC conversion run: | - wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/wp-content/storage/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword" + wget -O myfile.doc "http://localhost:6543/?url=https://file-examples.com/storage/fef4e75e176737761a179bf/2017/10/file-sample_100kB.odt&from=application/vnd.oasis.opendocument.text&to=application/msword" if file -b --mime-type ./myfile.doc |grep -q 'application/msword'; then echo "File is DOC"; @@ -225,7 +225,7 @@ jobs: name: debian-focal - name: Download docker image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "convertit:latest" From b531b5c8acac3ac66249d2e85a63610ba0a6d541 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 19 Nov 2024 09:01:40 +0100 Subject: [PATCH 11/15] update jobs --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99ee185..561f6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: continue-on-error: true name: Check flake8 rules steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -43,7 +43,7 @@ jobs: continue-on-error: true name: Check isort rules steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.8' From ecff2a72b6b921d874a4dd884b7b907630f6cf52 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 19 Nov 2024 10:53:32 +0100 Subject: [PATCH 12/15] fixes --- .docker/Dockerfile.debian.builder | 2 +- Dockerfile | 3 ++- debian/control | 1 + debian/rules | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.docker/Dockerfile.debian.builder b/.docker/Dockerfile.debian.builder index f414c8d..644264e 100644 --- a/.docker/Dockerfile.debian.builder +++ b/.docker/Dockerfile.debian.builder @@ -1,6 +1,6 @@ ARG DISTRO=ubuntu:focal -FROM ${DISTRO} as base +FROM ${DISTRO} AS base RUN apt-get update -qq -o Acquire::Languages=none && \ diff --git a/Dockerfile b/Dockerfile index a500d95..1375fb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ ARG DISTRO=jammy FROM ubuntu:${DISTRO} AS base LABEL org.opencontainers.image.authors="Makina Corpus " - RUN apt-get update && apt-get install -y -qq python3 libreoffice default-jre libreoffice-java-common inkscape python3-magic && \ apt-get autoclean && apt-get clean all && rm -rf /var/apt/lists/* @@ -39,3 +38,5 @@ COPY setup.py /opt/apps/convertit/setup.py COPY README.rst /opt/apps/convertit/README.rst RUN /opt/venv/bin/pip install . COPY production.ini /opt/apps/convertit/production.ini +VOLUME /var/cache/convertit/downloads +VOLUME /var/cache/convertit/converted \ No newline at end of file diff --git a/debian/control b/debian/control index 79abde3..d4fd300 100644 --- a/debian/control +++ b/debian/control @@ -16,6 +16,7 @@ Package: convertit Architecture: any Depends: ${misc:Depends}, + ${shlibs:Depends}, python3, python3-magic, inkscape, diff --git a/debian/rules b/debian/rules index 5297dbf..5a60b1b 100755 --- a/debian/rules +++ b/debian/rules @@ -23,4 +23,5 @@ override_dh_strip: dh_strip --no-automatic-dbgsym override_dh_shlibdeps: - dh_shlibdeps --exclude=/site-packages/ \ No newline at end of file + dh_shlibdeps \ + -X site-packages/Pillow.libs \ No newline at end of file From 46b93dfc835e040e0406a9271109278b8666003b Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 19 Nov 2024 11:22:38 +0100 Subject: [PATCH 13/15] fixes --- debian/rules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 5a60b1b..98664f9 100755 --- a/debian/rules +++ b/debian/rules @@ -12,7 +12,8 @@ override_dh_virtualenv: --upgrade-pip \ --preinstall wheel \ --preinstall setuptools \ - --builtin-venv \ + --builtin-venv + mkdir -p debian/convertit/usr/bin mkdir -p debian/convertit/opt/convertit cp production.ini debian/convertit/opt/convertit/convertit.ini From 48b144b2e8c9397703a0073113d30623e78d8789 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 19 Nov 2024 11:28:35 +0100 Subject: [PATCH 14/15] fixes --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 561f6f5..08e21e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,7 @@ jobs: name: Test ${{ matrix.distro }} runs-on: ubuntu-${{ matrix.version }} needs: [ build ] + continue-on-error: true steps: - uses: actions/download-artifact@v4 if: ${{ matrix.distro != 'docker' }} From 4566ffba02b91503d4b77100eca95acc1fc099a7 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 19 Nov 2024 12:06:41 +0100 Subject: [PATCH 15/15] fixes --- convertit/converters/unoconv.py | 11 +++++------ debian/postinst | 2 +- dev-requirements.txt | 11 +---------- requirements.in | 5 +++-- requirements.txt | 7 +++---- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/convertit/converters/unoconv.py b/convertit/converters/unoconv.py index 567bc47..ebd2ac1 100644 --- a/convertit/converters/unoconv.py +++ b/convertit/converters/unoconv.py @@ -3,7 +3,6 @@ import subprocess import tempfile from functools import partial -from mimetypes import types_map from convertit import exists @@ -68,9 +67,9 @@ def is_available(): def converters(): return { - (types_map['.csv'], types_map['.ods']): to_ods, - (types_map['.csv'], types_map['.xls']): to_xls, - (types_map['.ods'], types_map['.xls']): to_xls, - (types_map['.odt'], types_map['.doc']): to_doc, - (types_map['.odt'], types_map['.pdf']): to_pdf, + ('plain/text', 'application/vnd.oasis.opendocument.spreadsheet'): to_ods, + ('plain/text', 'application/vnd.ms-excel'): to_xls, + ('application/vnd.oasis.opendocument.spreadsheet', 'application/vnd.ms-excel'): to_xls, + ('application/vnd.oasis.opendocument.text', 'application/msword'): to_doc, + ('application/vnd.oasis.opendocument.text', 'application/pdf'): to_pdf, } diff --git a/debian/postinst b/debian/postinst index 526bbfb..8df90a1 100644 --- a/debian/postinst +++ b/debian/postinst @@ -2,6 +2,6 @@ adduser --system --group --home /run/convertit --no-create-home --quiet convertit || true mkdir -p /var/cache/convertit || true -chown convertit.convertit /var/cache/convertit || true +chown convertit:convertit /var/cache/convertit || true #DEBHELPER# diff --git a/dev-requirements.txt b/dev-requirements.txt index b6d1134..b8bbb84 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile dev-requirements.in @@ -16,8 +16,6 @@ flake8==6.1.0 # via -r dev-requirements.in freezegun==1.3.1 # via -r dev-requirements.in -importlib-metadata==7.0.0 - # via build isort==5.12.0 # via -r dev-requirements.in mccabe==0.7.0 @@ -42,11 +40,6 @@ six==1.16.0 # via python-dateutil soupsieve==2.5 # via beautifulsoup4 -tomli==2.0.1 - # via - # build - # pip-tools - # pyproject-hooks waitress==2.1.2 # via webtest webob==1.8.7 @@ -57,8 +50,6 @@ webtest==3.0.0 # via -r dev-requirements.in wheel==0.42.0 # via pip-tools -zipp==3.17.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements.in b/requirements.in index 34b1528..5b72317 100644 --- a/requirements.in +++ b/requirements.in @@ -1,3 +1,4 @@ -pyramid<=1.10.5 +pyramid python-magic -Pillow \ No newline at end of file +Pillow +gunicorn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index fecd6bc..daad2c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile +# pip-compile requirements.in # gunicorn==23.0.0 # via -r requirements.in hupper==1.12 # via pyramid -importlib-metadata packaging==23.2 # via gunicorn pastedeploy==3.1.0 @@ -21,7 +20,7 @@ plaster==1.1.2 # pyramid plaster-pastedeploy==1.0.1 # via pyramid -pyramid==1.10.5 +pyramid==2.0.2 # via -r requirements.in python-magic==0.4.27 # via -r requirements.in