Attempt to lock down dependencies

In the current world, we do not lock down dependencies at all. This means that developers are all potentially developing against moving targets, and some dependencies from which we subclass models could generate migrations (which could be backwards incompatible and cause issues for environments that use system packages pinned to older versions). In this commit is really 3 changes: - Update the updater script a bit - let it write out all the DAB application .txt files - Make pyproject.toml use those .txt files - Add current working/known upper bounds to all deps where possible before generating the .txt files. I believe this is the first step towards sanity here and vision into which versions of dependencies are getting pulled in and where. In the current world, we have a dep that released a new z-stream release which changed a field in a baseclass we're using. For developers who got that new version of the library (since we don't pin deps currently), it is easy to think that we need to push the new migration that gets generated. However we have no idea currently if that change would be compatible with apps using DAB, or apps forced to use the old version of the dependency due to system packages. Signed-off-by: Rick Elrod <[email protected]>
relrod · Aug 23, 2024 · 7658918 · 7658918
1 parent e533ddc
commit 7658918
Show file tree

Hide file tree

Showing 21 changed files with 318 additions and 89 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -33,28 +33,28 @@ dynamic = ["version", "dependencies", "optional-dependencies"]
 include = ["ansible_base*"]
 
 [tool.setuptools.dynamic]
-dependencies = {file = ["requirements/requirements.in"]}
+dependencies = {file = ["requirements/requirements.txt"]}
 optional-dependencies.all = { file = [
-    "requirements/requirements_activitystream.in",
-    "requirements/requirements_authentication.in",
-    "requirements/requirements_api_documentation.in",
-    "requirements/requirements_rest_filters.in",
-    "requirements/requirements_channels.in",
-    "requirements/requirements_jwt_consumer.in",
-    "requirements/requirements_testing.in",
-    "requirements/requirements_redis_client.in",
-    "requirements/requirements_oauth2_provider.in",
+    "requirements/requirements_activitystream.txt",
+    "requirements/requirements_authentication.txt",
+    "requirements/requirements_api_documentation.txt",
+    "requirements/requirements_rest_filters.txt",
+    "requirements/requirements_channels.txt",
+    "requirements/requirements_jwt_consumer.txt",
+    "requirements/requirements_testing.txt",
+    "requirements/requirements_redis_client.txt",
+    "requirements/requirements_oauth2_provider.txt",
 ] }
-optional-dependencies.activitystream = { file = [ "requirements/requirements_activitystream.in" ] }
-optional-dependencies.authentication = { file = [ "requirements/requirements_authentication.in" ] }
-optional-dependencies.api_documentation = { file = [ "requirements/requirements_api_documentation.in" ] }
-optional-dependencies.rest_filters = { file = [ "requirements/requirements_rest_filters.in" ] }
-optional-dependencies.rbac = { file = [ "requirements/requirements_rbac.in" ] }
-optional-dependencies.channel_auth = { file = [ "requirements/requirements_channels.in" ] }
-optional-dependencies.jwt_consumer = { file = [ "requirements/requirements_jwt_consumer.in" ] }
-optional-dependencies.testing = { file = [ "requirements/requirements_testing.in" ] }
-optional-dependencies.redis_client = { file = [ "requirements/requirements_redis_client.in" ] }
-optional-dependencies.oauth2_provider = { file = [ "requirements/requirements_oauth2_provider.in" ] }
+optional-dependencies.activitystream = { file = [ "requirements/requirements_activitystream.txt" ] }
+optional-dependencies.authentication = { file = [ "requirements/requirements_authentication.txt" ] }
+optional-dependencies.api_documentation = { file = [ "requirements/requirements_api_documentation.txt" ] }
+optional-dependencies.rest_filters = { file = [ "requirements/requirements_rest_filters.txt" ] }
+optional-dependencies.rbac = { file = [ "requirements/requirements_rbac.txt" ] }
+optional-dependencies.channel_auth = { file = [ "requirements/requirements_channels.txt" ] }
+optional-dependencies.jwt_consumer = { file = [ "requirements/requirements_jwt_consumer.txt" ] }
+optional-dependencies.testing = { file = [ "requirements/requirements_testing.txt" ] }
+optional-dependencies.redis_client = { file = [ "requirements/requirements_redis_client.txt" ] }
+optional-dependencies.oauth2_provider = { file = [ "requirements/requirements_oauth2_provider.txt" ] }
 
 [build-system]
 requires = ["setuptools>=64", "setuptools_scm>=8"]

diff --git a/requirements/requirements.in b/requirements/requirements.in
@@ -2,9 +2,9 @@
 # Only generic requirements for django-ansible-base (or the common feature) should be listed here.
 # if you are add a new feature which requires dependencies they should be in a separate requirements_<feature>.in file
 #
-cryptography
+cryptography<=42.0.5
 Django>=4.2.5,<4.3.0
-djangorestframework
-django-crum
-django-split-settings
-inflection
+djangorestframework<=3.15.1
+django-crum<=0.7.9
+django-split-settings<=1.2.0
+inflection<=0.5.1
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
@@ -0,0 +1,23 @@
+asgiref==3.8.1
+    # via django
+cffi==1.17.0
+    # via cryptography
+cryptography==42.0.5
+    # via -r requirements/requirements.in
+django==4.2.15
+    # via
+    #   -r requirements/requirements.in
+    #   django-crum
+    #   djangorestframework
+django-crum==0.7.9
+    # via -r requirements/requirements.in
+django-split-settings==1.2.0
+    # via -r requirements/requirements.in
+djangorestframework==3.15.1
+    # via -r requirements/requirements.in
+inflection==0.5.1
+    # via -r requirements/requirements.in
+pycparser==2.22
+    # via cffi
+sqlparse==0.5.1
+    # via django
diff --git a/requirements/requirements_activitystream.txt b/requirements/requirements_activitystream.txt
@@ -0,0 +1 @@
+
diff --git a/requirements/requirements_all.txt b/requirements/requirements_all.txt
@@ -2,15 +2,15 @@ asgiref==3.8.1
     # via
     #   channels
     #   django
-attrs==23.2.0
+attrs==24.2.0
     # via
     #   jsonschema
     #   referencing
-certifi==2024.2.2
+certifi==2024.7.4
     # via requests
-cffi==1.16.0
+cffi==1.17.0
     # via cryptography
-channels==4.0.0
+channels==4.1.0
     # via -r requirements/requirements_channels.in
 charset-normalizer==3.3.2
     # via requests
@@ -24,7 +24,7 @@ defusedxml==0.8.0rc2
     # via
     #   python3-openid
     #   social-auth-core
-django==4.2.11
+django==4.2.15
     # via
     #   -r requirements/requirements.in
     #   channels
@@ -35,23 +35,23 @@ django==4.2.11
     #   djangorestframework
     #   drf-spectacular
     #   social-auth-app-django
-django-auth-ldap==4.7.0
+django-auth-ldap==4.0.0
     # via -r requirements/requirements_authentication.in
 django-crum==0.7.9
     # via -r requirements/requirements.in
 django-oauth-toolkit==2.3.0
     # via -r requirements/requirements_oauth2_provider.in
 django-redis==5.4.0
     # via -r requirements/requirements_redis_client.in
-django-split-settings==1.3.0
+django-split-settings==1.2.0
     # via -r requirements/requirements.in
 djangorestframework==3.15.1
     # via
     #   -r requirements/requirements.in
     #   drf-spectacular
-drf-spectacular==0.27.1
+drf-spectacular==0.26.5
     # via -r requirements/requirements_api_documentation.in
-idna==3.6
+idna==3.8
     # via requests
 inflection==0.5.1
     # via
@@ -61,42 +61,42 @@ iniconfig==2.0.0
     # via pytest
 isodate==0.6.1
     # via python3-saml
-jsonschema==4.21.1
+jsonschema==4.23.0
     # via drf-spectacular
 jsonschema-specifications==2023.12.1
     # via jsonschema
 jwcrypto==1.5.6
     # via django-oauth-toolkit
-lxml==5.1.0
+lxml==5.3.0
     # via
     #   python3-saml
     #   xmlsec
-netaddr==1.2.1
+netaddr==1.3.0
     # via pyrad
 oauthlib==3.2.2
     # via
     #   django-oauth-toolkit
     #   requests-oauthlib
     #   social-auth-core
-packaging==24.0
+packaging==24.1
     # via pytest
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pyasn1==0.5.1
+pyasn1==0.6.0
     # via
     #   pyasn1-modules
     #   python-ldap
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.0
     # via python-ldap
-pycparser==2.21
+pycparser==2.22
     # via cffi
-pyjwt==2.8.0
+pyjwt==2.7.0
     # via
     #   -r requirements/requirements_jwt_consumer.in
     #   social-auth-core
 pyrad==2.4
     # via -r requirements/requirements_authentication.in
-pytest==8.1.1
+pytest==8.3.2
     # via
     #   -r requirements/requirements_testing.in
     #   pytest-django
@@ -110,13 +110,13 @@ python3-openid==3.2.0
     # via social-auth-core
 python3-saml==1.16.0
     # via -r requirements/requirements_authentication.in
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via drf-spectacular
-redis==5.0.3
+redis==4.6.0
     # via
     #   -r requirements/requirements_redis_client.in
     #   django-redis
-referencing==0.34.0
+referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
@@ -128,7 +128,7 @@ requests==2.31.0
     #   social-auth-core
 requests-oauthlib==2.0.0
     # via social-auth-core
-rpds-py==0.18.0
+rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
@@ -137,21 +137,21 @@ six==1.16.0
     #   isodate
     #   pyrad
     #   tacacs-plus
-social-auth-app-django==5.4.0
+social-auth-app-django==5.4.1
     # via -r requirements/requirements_authentication.in
-social-auth-core==4.5.3
+social-auth-core==4.5.4
     # via social-auth-app-django
-sqlparse==0.4.4
+sqlparse==0.5.1
     # via django
 tabulate==0.9.0
     # via -r requirements/requirements_authentication.in
 tacacs-plus==2.6
     # via -r requirements/requirements_authentication.in
-typing-extensions==4.11.0
+typing-extensions==4.12.2
     # via jwcrypto
 uritemplate==4.1.1
     # via drf-spectacular
-urllib3==2.2.1
+urllib3==2.2.2
     # via requests
 xmlsec==1.3.13
     # via

diff --git a/requirements/requirements_api_documentation.in b/requirements/requirements_api_documentation.in
@@ -1 +1 @@
-drf-spectacular
+drf-spectacular<=0.26.5
diff --git a/requirements/requirements_api_documentation.txt b/requirements/requirements_api_documentation.txt
@@ -0,0 +1,34 @@
+asgiref==3.8.1
+    # via django
+attrs==24.2.0
+    # via
+    #   jsonschema
+    #   referencing
+django==5.1
+    # via
+    #   djangorestframework
+    #   drf-spectacular
+djangorestframework==3.15.2
+    # via drf-spectacular
+drf-spectacular==0.26.5
+    # via -r requirements/requirements_api_documentation.in
+inflection==0.5.1
+    # via drf-spectacular
+jsonschema==4.23.0
+    # via drf-spectacular
+jsonschema-specifications==2023.12.1
+    # via jsonschema
+pyyaml==6.0.2
+    # via drf-spectacular
+referencing==0.35.1
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+rpds-py==0.20.0
+    # via
+    #   jsonschema
+    #   referencing
+sqlparse==0.5.1
+    # via django
+uritemplate==4.1.1
+    # via drf-spectacular
diff --git a/requirements/requirements_authentication.in b/requirements/requirements_authentication.in
@@ -1,17 +1,17 @@
-social-auth-app-django
-tabulate
+social-auth-app-django<=5.4.1
+tabulate<=0.9.0
 
 # These should eventually be split out when the authentications move into their own repo
 
 # LDAP Authenticator Plugins
-django-auth-ldap
-python-ldap
+django-auth-ldap<=4.0.0
+python-ldap<=3.4.4
 
 # Social Authenticator Plugins
-python3-saml
-tacacs_plus
+python3-saml<=1.16.0
+tacacs_plus<=2.6
 
 xmlsec==1.3.13 # Pin for https://github.com/xmlsec/python-xmlsec/issues/314
 
 # RADIUS Authenticator Plugin
-pyrad
+pyrad<=2.4
diff --git a/requirements/requirements_authentication.txt b/requirements/requirements_authentication.txt
@@ -0,0 +1,81 @@
+asgiref==3.8.1
+    # via django
+certifi==2024.7.4
+    # via requests
+cffi==1.17.0
+    # via cryptography
+charset-normalizer==3.3.2
+    # via requests
+cryptography==43.0.0
+    # via social-auth-core
+defusedxml==0.8.0rc2
+    # via
+    #   python3-openid
+    #   social-auth-core
+django==5.1
+    # via
+    #   django-auth-ldap
+    #   social-auth-app-django
+django-auth-ldap==4.0.0
+    # via -r requirements/requirements_authentication.in
+idna==3.8
+    # via requests
+isodate==0.6.1
+    # via python3-saml
+lxml==5.3.0
+    # via
+    #   python3-saml
+    #   xmlsec
+netaddr==1.3.0
+    # via pyrad
+oauthlib==3.2.2
+    # via
+    #   requests-oauthlib
+    #   social-auth-core
+pyasn1==0.6.0
+    # via
+    #   pyasn1-modules
+    #   python-ldap
+pyasn1-modules==0.4.0
+    # via python-ldap
+pycparser==2.22
+    # via cffi
+pyjwt==2.9.0
+    # via social-auth-core
+pyrad==2.4
+    # via -r requirements/requirements_authentication.in
+python-ldap==3.4.4
+    # via
+    #   -r requirements/requirements_authentication.in
+    #   django-auth-ldap
+python3-openid==3.2.0
+    # via social-auth-core
+python3-saml==1.16.0
+    # via -r requirements/requirements_authentication.in
+requests==2.32.3
+    # via
+    #   requests-oauthlib
+    #   social-auth-core
+requests-oauthlib==2.0.0
+    # via social-auth-core
+six==1.16.0
+    # via
+    #   isodate
+    #   pyrad
+    #   tacacs-plus
+social-auth-app-django==5.4.1
+    # via -r requirements/requirements_authentication.in
+social-auth-core==4.5.4
+    # via social-auth-app-django
+sqlparse==0.5.1
+    # via django
+tabulate==0.9.0
+    # via -r requirements/requirements_authentication.in
+tacacs-plus==2.6
+    # via -r requirements/requirements_authentication.in
+urllib3==2.2.2
+    # via requests
+xmlsec==1.3.13
+    # via
+    #   -r requirements/requirements_authentication.in
+    #   python3-saml