From 7658918b7250b34e3deb3ac9d0624badcc7a68a8 Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Fri, 23 Aug 2024 16:01:20 +0200 Subject: [PATCH] Attempt to lock down dependencies In the current world, we do not lock down dependencies at all. This means that developers are all potentially developing against moving targets, and some dependencies from which we subclass models could generate migrations (which could be backwards incompatible and cause issues for environments that use system packages pinned to older versions). In this commit is really 3 changes: - Update the updater script a bit - let it write out all the DAB application .txt files - Make pyproject.toml use those .txt files - Add current working/known upper bounds to all deps where possible before generating the .txt files. I believe this is the first step towards sanity here and vision into which versions of dependencies are getting pulled in and where. In the current world, we have a dep that released a new z-stream release which changed a field in a baseclass we're using. For developers who got that new version of the library (since we don't pin deps currently), it is easy to think that we need to push the new migration that gets generated. However we have no idea currently if that change would be compatible with apps using DAB, or apps forced to use the old version of the dependency due to system packages. Signed-off-by: Rick Elrod --- pyproject.toml | 40 ++++----- requirements/requirements.in | 10 +-- requirements/requirements.txt | 23 ++++++ requirements/requirements_activitystream.txt | 1 + requirements/requirements_all.txt | 56 ++++++------- .../requirements_api_documentation.in | 2 +- .../requirements_api_documentation.txt | 34 ++++++++ requirements/requirements_authentication.in | 14 ++-- requirements/requirements_authentication.txt | 81 +++++++++++++++++++ requirements/requirements_channels.txt | 10 +++ requirements/requirements_dev.txt | 8 +- requirements/requirements_jwt_consumer.in | 4 +- requirements/requirements_jwt_consumer.txt | 12 +++ requirements/requirements_oauth2_provider.txt | 30 +++++++ requirements/requirements_rbac.txt | 1 + requirements/requirements_redis_client.in | 4 +- requirements/requirements_redis_client.txt | 12 +++ requirements/requirements_rest_filters.txt | 1 + requirements/requirements_testing.in | 2 +- requirements/requirements_testing.txt | 18 +++++ requirements/updater.sh | 44 +++++----- 21 files changed, 318 insertions(+), 89 deletions(-) create mode 100644 requirements/requirements.txt create mode 100644 requirements/requirements_activitystream.txt create mode 100644 requirements/requirements_api_documentation.txt create mode 100644 requirements/requirements_authentication.txt create mode 100644 requirements/requirements_channels.txt create mode 100644 requirements/requirements_jwt_consumer.txt create mode 100644 requirements/requirements_oauth2_provider.txt create mode 100644 requirements/requirements_rbac.txt create mode 100644 requirements/requirements_redis_client.txt create mode 100644 requirements/requirements_rest_filters.txt create mode 100644 requirements/requirements_testing.txt diff --git a/pyproject.toml b/pyproject.toml index 3333372f0..88b9c1598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,28 +33,28 @@ dynamic = ["version", "dependencies", "optional-dependencies"] include = ["ansible_base*"] [tool.setuptools.dynamic] -dependencies = {file = ["requirements/requirements.in"]} +dependencies = {file = ["requirements/requirements.txt"]} optional-dependencies.all = { file = [ - "requirements/requirements_activitystream.in", - "requirements/requirements_authentication.in", - "requirements/requirements_api_documentation.in", - "requirements/requirements_rest_filters.in", - "requirements/requirements_channels.in", - "requirements/requirements_jwt_consumer.in", - "requirements/requirements_testing.in", - "requirements/requirements_redis_client.in", - "requirements/requirements_oauth2_provider.in", + "requirements/requirements_activitystream.txt", + "requirements/requirements_authentication.txt", + "requirements/requirements_api_documentation.txt", + "requirements/requirements_rest_filters.txt", + "requirements/requirements_channels.txt", + "requirements/requirements_jwt_consumer.txt", + "requirements/requirements_testing.txt", + "requirements/requirements_redis_client.txt", + "requirements/requirements_oauth2_provider.txt", ] } -optional-dependencies.activitystream = { file = [ "requirements/requirements_activitystream.in" ] } -optional-dependencies.authentication = { file = [ "requirements/requirements_authentication.in" ] } -optional-dependencies.api_documentation = { file = [ "requirements/requirements_api_documentation.in" ] } -optional-dependencies.rest_filters = { file = [ "requirements/requirements_rest_filters.in" ] } -optional-dependencies.rbac = { file = [ "requirements/requirements_rbac.in" ] } -optional-dependencies.channel_auth = { file = [ "requirements/requirements_channels.in" ] } -optional-dependencies.jwt_consumer = { file = [ "requirements/requirements_jwt_consumer.in" ] } -optional-dependencies.testing = { file = [ "requirements/requirements_testing.in" ] } -optional-dependencies.redis_client = { file = [ "requirements/requirements_redis_client.in" ] } -optional-dependencies.oauth2_provider = { file = [ "requirements/requirements_oauth2_provider.in" ] } +optional-dependencies.activitystream = { file = [ "requirements/requirements_activitystream.txt" ] } +optional-dependencies.authentication = { file = [ "requirements/requirements_authentication.txt" ] } +optional-dependencies.api_documentation = { file = [ "requirements/requirements_api_documentation.txt" ] } +optional-dependencies.rest_filters = { file = [ "requirements/requirements_rest_filters.txt" ] } +optional-dependencies.rbac = { file = [ "requirements/requirements_rbac.txt" ] } +optional-dependencies.channel_auth = { file = [ "requirements/requirements_channels.txt" ] } +optional-dependencies.jwt_consumer = { file = [ "requirements/requirements_jwt_consumer.txt" ] } +optional-dependencies.testing = { file = [ "requirements/requirements_testing.txt" ] } +optional-dependencies.redis_client = { file = [ "requirements/requirements_redis_client.txt" ] } +optional-dependencies.oauth2_provider = { file = [ "requirements/requirements_oauth2_provider.txt" ] } [build-system] requires = ["setuptools>=64", "setuptools_scm>=8"] diff --git a/requirements/requirements.in b/requirements/requirements.in index 42214361f..46f92eb38 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -2,9 +2,9 @@ # Only generic requirements for django-ansible-base (or the common feature) should be listed here. # if you are add a new feature which requires dependencies they should be in a separate requirements_.in file # -cryptography +cryptography<=42.0.5 Django>=4.2.5,<4.3.0 -djangorestframework -django-crum -django-split-settings -inflection +djangorestframework<=3.15.1 +django-crum<=0.7.9 +django-split-settings<=1.2.0 +inflection<=0.5.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 000000000..0341c5c32 --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,23 @@ +asgiref==3.8.1 + # via django +cffi==1.17.0 + # via cryptography +cryptography==42.0.5 + # via -r requirements/requirements.in +django==4.2.15 + # via + # -r requirements/requirements.in + # django-crum + # djangorestframework +django-crum==0.7.9 + # via -r requirements/requirements.in +django-split-settings==1.2.0 + # via -r requirements/requirements.in +djangorestframework==3.15.1 + # via -r requirements/requirements.in +inflection==0.5.1 + # via -r requirements/requirements.in +pycparser==2.22 + # via cffi +sqlparse==0.5.1 + # via django diff --git a/requirements/requirements_activitystream.txt b/requirements/requirements_activitystream.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/requirements/requirements_activitystream.txt @@ -0,0 +1 @@ + diff --git a/requirements/requirements_all.txt b/requirements/requirements_all.txt index 27bcbacf5..9553e9af6 100644 --- a/requirements/requirements_all.txt +++ b/requirements/requirements_all.txt @@ -2,15 +2,15 @@ asgiref==3.8.1 # via # channels # django -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -certifi==2024.2.2 +certifi==2024.7.4 # via requests -cffi==1.16.0 +cffi==1.17.0 # via cryptography -channels==4.0.0 +channels==4.1.0 # via -r requirements/requirements_channels.in charset-normalizer==3.3.2 # via requests @@ -24,7 +24,7 @@ defusedxml==0.8.0rc2 # via # python3-openid # social-auth-core -django==4.2.11 +django==4.2.15 # via # -r requirements/requirements.in # channels @@ -35,7 +35,7 @@ django==4.2.11 # djangorestframework # drf-spectacular # social-auth-app-django -django-auth-ldap==4.7.0 +django-auth-ldap==4.0.0 # via -r requirements/requirements_authentication.in django-crum==0.7.9 # via -r requirements/requirements.in @@ -43,15 +43,15 @@ django-oauth-toolkit==2.3.0 # via -r requirements/requirements_oauth2_provider.in django-redis==5.4.0 # via -r requirements/requirements_redis_client.in -django-split-settings==1.3.0 +django-split-settings==1.2.0 # via -r requirements/requirements.in djangorestframework==3.15.1 # via # -r requirements/requirements.in # drf-spectacular -drf-spectacular==0.27.1 +drf-spectacular==0.26.5 # via -r requirements/requirements_api_documentation.in -idna==3.6 +idna==3.8 # via requests inflection==0.5.1 # via @@ -61,42 +61,42 @@ iniconfig==2.0.0 # via pytest isodate==0.6.1 # via python3-saml -jsonschema==4.21.1 +jsonschema==4.23.0 # via drf-spectacular jsonschema-specifications==2023.12.1 # via jsonschema jwcrypto==1.5.6 # via django-oauth-toolkit -lxml==5.1.0 +lxml==5.3.0 # via # python3-saml # xmlsec -netaddr==1.2.1 +netaddr==1.3.0 # via pyrad oauthlib==3.2.2 # via # django-oauth-toolkit # requests-oauthlib # social-auth-core -packaging==24.0 +packaging==24.1 # via pytest -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # python-ldap -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via python-ldap -pycparser==2.21 +pycparser==2.22 # via cffi -pyjwt==2.8.0 +pyjwt==2.7.0 # via # -r requirements/requirements_jwt_consumer.in # social-auth-core pyrad==2.4 # via -r requirements/requirements_authentication.in -pytest==8.1.1 +pytest==8.3.2 # via # -r requirements/requirements_testing.in # pytest-django @@ -110,13 +110,13 @@ python3-openid==3.2.0 # via social-auth-core python3-saml==1.16.0 # via -r requirements/requirements_authentication.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via drf-spectacular -redis==5.0.3 +redis==4.6.0 # via # -r requirements/requirements_redis_client.in # django-redis -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -128,7 +128,7 @@ requests==2.31.0 # social-auth-core requests-oauthlib==2.0.0 # via social-auth-core -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing @@ -137,21 +137,21 @@ six==1.16.0 # isodate # pyrad # tacacs-plus -social-auth-app-django==5.4.0 +social-auth-app-django==5.4.1 # via -r requirements/requirements_authentication.in -social-auth-core==4.5.3 +social-auth-core==4.5.4 # via social-auth-app-django -sqlparse==0.4.4 +sqlparse==0.5.1 # via django tabulate==0.9.0 # via -r requirements/requirements_authentication.in tacacs-plus==2.6 # via -r requirements/requirements_authentication.in -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via jwcrypto uritemplate==4.1.1 # via drf-spectacular -urllib3==2.2.1 +urllib3==2.2.2 # via requests xmlsec==1.3.13 # via diff --git a/requirements/requirements_api_documentation.in b/requirements/requirements_api_documentation.in index b4dbe94bd..f9e7071bf 100644 --- a/requirements/requirements_api_documentation.in +++ b/requirements/requirements_api_documentation.in @@ -1 +1 @@ -drf-spectacular +drf-spectacular<=0.26.5 diff --git a/requirements/requirements_api_documentation.txt b/requirements/requirements_api_documentation.txt new file mode 100644 index 000000000..f81ecc120 --- /dev/null +++ b/requirements/requirements_api_documentation.txt @@ -0,0 +1,34 @@ +asgiref==3.8.1 + # via django +attrs==24.2.0 + # via + # jsonschema + # referencing +django==5.1 + # via + # djangorestframework + # drf-spectacular +djangorestframework==3.15.2 + # via drf-spectacular +drf-spectacular==0.26.5 + # via -r requirements/requirements_api_documentation.in +inflection==0.5.1 + # via drf-spectacular +jsonschema==4.23.0 + # via drf-spectacular +jsonschema-specifications==2023.12.1 + # via jsonschema +pyyaml==6.0.2 + # via drf-spectacular +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.20.0 + # via + # jsonschema + # referencing +sqlparse==0.5.1 + # via django +uritemplate==4.1.1 + # via drf-spectacular diff --git a/requirements/requirements_authentication.in b/requirements/requirements_authentication.in index 8c4620b75..b4b145845 100644 --- a/requirements/requirements_authentication.in +++ b/requirements/requirements_authentication.in @@ -1,17 +1,17 @@ -social-auth-app-django -tabulate +social-auth-app-django<=5.4.1 +tabulate<=0.9.0 # These should eventually be split out when the authentications move into their own repo # LDAP Authenticator Plugins -django-auth-ldap -python-ldap +django-auth-ldap<=4.0.0 +python-ldap<=3.4.4 # Social Authenticator Plugins -python3-saml -tacacs_plus +python3-saml<=1.16.0 +tacacs_plus<=2.6 xmlsec==1.3.13 # Pin for https://github.com/xmlsec/python-xmlsec/issues/314 # RADIUS Authenticator Plugin -pyrad +pyrad<=2.4 diff --git a/requirements/requirements_authentication.txt b/requirements/requirements_authentication.txt new file mode 100644 index 000000000..8c43e763f --- /dev/null +++ b/requirements/requirements_authentication.txt @@ -0,0 +1,81 @@ +asgiref==3.8.1 + # via django +certifi==2024.7.4 + # via requests +cffi==1.17.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +cryptography==43.0.0 + # via social-auth-core +defusedxml==0.8.0rc2 + # via + # python3-openid + # social-auth-core +django==5.1 + # via + # django-auth-ldap + # social-auth-app-django +django-auth-ldap==4.0.0 + # via -r requirements/requirements_authentication.in +idna==3.8 + # via requests +isodate==0.6.1 + # via python3-saml +lxml==5.3.0 + # via + # python3-saml + # xmlsec +netaddr==1.3.0 + # via pyrad +oauthlib==3.2.2 + # via + # requests-oauthlib + # social-auth-core +pyasn1==0.6.0 + # via + # pyasn1-modules + # python-ldap +pyasn1-modules==0.4.0 + # via python-ldap +pycparser==2.22 + # via cffi +pyjwt==2.9.0 + # via social-auth-core +pyrad==2.4 + # via -r requirements/requirements_authentication.in +python-ldap==3.4.4 + # via + # -r requirements/requirements_authentication.in + # django-auth-ldap +python3-openid==3.2.0 + # via social-auth-core +python3-saml==1.16.0 + # via -r requirements/requirements_authentication.in +requests==2.32.3 + # via + # requests-oauthlib + # social-auth-core +requests-oauthlib==2.0.0 + # via social-auth-core +six==1.16.0 + # via + # isodate + # pyrad + # tacacs-plus +social-auth-app-django==5.4.1 + # via -r requirements/requirements_authentication.in +social-auth-core==4.5.4 + # via social-auth-app-django +sqlparse==0.5.1 + # via django +tabulate==0.9.0 + # via -r requirements/requirements_authentication.in +tacacs-plus==2.6 + # via -r requirements/requirements_authentication.in +urllib3==2.2.2 + # via requests +xmlsec==1.3.13 + # via + # -r requirements/requirements_authentication.in + # python3-saml diff --git a/requirements/requirements_channels.txt b/requirements/requirements_channels.txt new file mode 100644 index 000000000..ebf6dd22d --- /dev/null +++ b/requirements/requirements_channels.txt @@ -0,0 +1,10 @@ +asgiref==3.8.1 + # via + # channels + # django +channels==4.1.0 + # via -r requirements/requirements_channels.in +django==5.1 + # via channels +sqlparse==0.5.1 + # via django diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index 4e4ab407b..83ca3d5d2 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -2,9 +2,9 @@ ansible # Used in build process to generate some configs build Django>=4.2.5,<4.3.0 django-debug-toolbar -django-extensions -djangorestframework -django-split-settings +django-extensions<=3.2.3 +djangorestframework<=3.15.1 +django-split-settings<=1.2.0 ipython tox typeguard @@ -14,5 +14,5 @@ pytest-xdist pytest-cov pytest-django setuptools-scm -psycopg[binary] +psycopg[binary]<=3.1.17 sdb diff --git a/requirements/requirements_jwt_consumer.in b/requirements/requirements_jwt_consumer.in index 4478fcab2..31d9b9a0c 100644 --- a/requirements/requirements_jwt_consumer.in +++ b/requirements/requirements_jwt_consumer.in @@ -1,2 +1,2 @@ -pyjwt -requests +pyjwt<=2.7.0 +requests<=2.31.0 diff --git a/requirements/requirements_jwt_consumer.txt b/requirements/requirements_jwt_consumer.txt new file mode 100644 index 000000000..00f6cc605 --- /dev/null +++ b/requirements/requirements_jwt_consumer.txt @@ -0,0 +1,12 @@ +certifi==2024.7.4 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.8 + # via requests +pyjwt==2.7.0 + # via -r requirements/requirements_jwt_consumer.in +requests==2.31.0 + # via -r requirements/requirements_jwt_consumer.in +urllib3==2.2.2 + # via requests diff --git a/requirements/requirements_oauth2_provider.txt b/requirements/requirements_oauth2_provider.txt new file mode 100644 index 000000000..350e96519 --- /dev/null +++ b/requirements/requirements_oauth2_provider.txt @@ -0,0 +1,30 @@ +asgiref==3.8.1 + # via django +certifi==2024.7.4 + # via requests +cffi==1.17.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +cryptography==43.0.0 + # via jwcrypto +django==5.1 + # via django-oauth-toolkit +django-oauth-toolkit==2.3.0 + # via -r requirements/requirements_oauth2_provider.in +idna==3.8 + # via requests +jwcrypto==1.5.6 + # via django-oauth-toolkit +oauthlib==3.2.2 + # via django-oauth-toolkit +pycparser==2.22 + # via cffi +requests==2.32.3 + # via django-oauth-toolkit +sqlparse==0.5.1 + # via django +typing-extensions==4.12.2 + # via jwcrypto +urllib3==2.2.2 + # via requests diff --git a/requirements/requirements_rbac.txt b/requirements/requirements_rbac.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/requirements/requirements_rbac.txt @@ -0,0 +1 @@ + diff --git a/requirements/requirements_redis_client.in b/requirements/requirements_redis_client.in index 37caa8f1b..b5c9b0f7a 100644 --- a/requirements/requirements_redis_client.in +++ b/requirements/requirements_redis_client.in @@ -1,2 +1,2 @@ -django-redis -redis +django-redis<=5.4.0 +redis<=4.6.0 diff --git a/requirements/requirements_redis_client.txt b/requirements/requirements_redis_client.txt new file mode 100644 index 000000000..37af583e9 --- /dev/null +++ b/requirements/requirements_redis_client.txt @@ -0,0 +1,12 @@ +asgiref==3.8.1 + # via django +django==5.1 + # via django-redis +django-redis==5.4.0 + # via -r requirements/requirements_redis_client.in +redis==4.6.0 + # via + # -r requirements/requirements_redis_client.in + # django-redis +sqlparse==0.5.1 + # via django diff --git a/requirements/requirements_rest_filters.txt b/requirements/requirements_rest_filters.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/requirements/requirements_rest_filters.txt @@ -0,0 +1 @@ + diff --git a/requirements/requirements_testing.in b/requirements/requirements_testing.in index 5c1f36223..265c12799 100644 --- a/requirements/requirements_testing.in +++ b/requirements/requirements_testing.in @@ -1,5 +1,5 @@ # NOTE: Only dependencies needed to use ansible_base.lib.testing.* should go here. # Deps specific to django-ansible-base tests should go in requirements_dev.txt -cryptography +cryptography<=42.0.5 pytest pytest-django diff --git a/requirements/requirements_testing.txt b/requirements/requirements_testing.txt new file mode 100644 index 000000000..c069ca747 --- /dev/null +++ b/requirements/requirements_testing.txt @@ -0,0 +1,18 @@ +cffi==1.17.0 + # via cryptography +cryptography==42.0.5 + # via -r requirements/requirements_testing.in +iniconfig==2.0.0 + # via pytest +packaging==24.1 + # via pytest +pluggy==1.5.0 + # via pytest +pycparser==2.22 + # via cffi +pytest==8.3.2 + # via + # -r requirements/requirements_testing.in + # pytest-django +pytest-django==4.8.0 + # via -r requirements/requirements_testing.in diff --git a/requirements/updater.sh b/requirements/updater.sh index 7c48c7428..42f135459 100755 --- a/requirements/updater.sh +++ b/requirements/updater.sh @@ -1,62 +1,69 @@ #!/usr/bin/env bash +# -*- coding: utf-8; mode: sh; sh-indentation: 2; indent-tabs-mode: nil; sh-basic-offset: 2; -*- + set -ue PYTHON=python3.11 for FILE in requirements.in requirements_all.txt ; do - if [ ! -f ${FILE} ] ; then - touch ${FILE} - fi + if [ ! -f ${FILE} ] ; then + touch ${FILE} + fi done requirements_dir="$(readlink -f .)" requirements_txt="$(readlink -f ./requirements_all.txt)" pip_compile="pip-compile --no-header --quiet -r --allow-unsafe" _cleanup() { - cd / test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}" } generate_requirements() { - venv="`pwd`/venv" - echo $venv + venv="$(pwd)/venv" + echo "Using virtual environment: $venv" ${PYTHON} -m venv "${venv}" # shellcheck disable=SC1090 source ${venv}/bin/activate - ${venv}/bin/python -m pip install -U 'pip' pip-tools + ${venv}/bin/pip install -U pip pip-tools + + ${pip_compile} ${requirements_dir}/requirements*.in --output-file requirements_all.txt - ${pip_compile} $(ls ${requirements_dir}/requirements*.in | xargs) --output-file requirements_all.txt + for file in ${requirements_dir}/requirements*.in; do + app="$(basename $file .in)" + echo "Compiling deps for $app" + ${pip_compile} $file --output-file "$app.txt" + done } main() { base_dir=$(pwd) - _tmp=$(${PYTHON} -c "import tempfile; print(tempfile.mkdtemp(suffix='.ansible_base-requirements', dir='/tmp'))") + _tmp=$(mktemp -d /tmp/tmp.XXXXXX.ansible_base-requirements) trap _cleanup INT TERM EXIT case $1 in "run") NEEDS_HELP=0 - ;; + ;; "upgrade") NEEDS_HELP=0 pip_compile="${pip_compile} --upgrade" - ;; + ;; "help") NEEDS_HELP=1 - ;; + ;; *) echo "" echo "ERROR: Parameter $1 not valid" echo "" NEEDS_HELP=1 - ;; - esac + ;; + esac if [[ "$NEEDS_HELP" == "1" ]] ; then - echo "This script generates requirements_all.txt from requirements[_*].in" + echo "This script generates pinned requirements.txt files from requirements[_*].in" echo "" echo "Usage: $0 [run|upgrade]" echo "" @@ -68,17 +75,16 @@ main() { exit fi - cp -vf ${requirements_txt} "${_tmp}" cd "${_tmp}" - generate_requirements echo "Changing $base_dir to requirements" - cat requirements_all.txt | sed "s:$base_dir:requirements:" > "${requirements_txt}" + for file in requirements*.txt; do + sed "s:$base_dir:requirements:" "$file" > "$requirements_dir/$file" + done _cleanup } # set EVAL=1 in case you want to source this script test "${EVAL:-0}" -eq "1" || main "${1:-}" -