From 2d7bbc4ec83d2f6ea8ff396c9dc817925501e238 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 10 Jan 2025 10:33:35 -0500 Subject: [PATCH 1/2] AAP-37080 Delete the cleanup_tokens system job template (#15711) Delete the cleanup_tokens system job template --- .../0078_v360_clear_sessions_tokens_jt.py | 4 +- .../0200_delete_token_cleanup_job.py | 44 ++++++++++++++ awx/main/migrations/_create_system_jobs.py | 35 ++--------- awx/main/models/jobs.py | 1 - .../test_inventory_source_migration.py | 0 .../test_org_admin_migration.py | 0 .../migrations/test_rbac_migration.py | 49 ++++++++++++++++ .../migrations/test_token_sjt_removal.py | 58 +++++++++++++++++++ 8 files changed, 158 insertions(+), 33 deletions(-) create mode 100644 awx/main/migrations/0200_delete_token_cleanup_job.py rename awx/main/tests/functional/{ => migrations}/test_inventory_source_migration.py (100%) rename awx/main/tests/functional/{ => migrations}/test_org_admin_migration.py (100%) create mode 100644 awx/main/tests/functional/migrations/test_rbac_migration.py create mode 100644 awx/main/tests/functional/migrations/test_token_sjt_removal.py diff --git a/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py b/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py index 630a4b38ef5a..130cfd8bd3fb 100644 --- a/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py +++ b/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals from django.db import migrations, models -from awx.main.migrations._create_system_jobs import create_clearsessions_jt, create_cleartokens_jt +from awx.main.migrations._create_system_jobs import create_clearsessions_jt class Migration(migrations.Migration): @@ -14,7 +14,7 @@ class Migration(migrations.Migration): operations = [ # Schedule Analytics System Job Template migrations.RunPython(create_clearsessions_jt, migrations.RunPython.noop), - migrations.RunPython(create_cleartokens_jt, migrations.RunPython.noop), + # previously ran create_cleartokens_jt, logic for this has been removed migrations.AlterField( model_name='systemjob', name='job_type', diff --git a/awx/main/migrations/0200_delete_token_cleanup_job.py b/awx/main/migrations/0200_delete_token_cleanup_job.py new file mode 100644 index 000000000000..b9fe4c240132 --- /dev/null +++ b/awx/main/migrations/0200_delete_token_cleanup_job.py @@ -0,0 +1,44 @@ +# Generated by Django 4.2.16 on 2024-12-18 16:05 + +from django.db import migrations, models + +from awx.main.migrations._create_system_jobs import delete_clear_tokens_sjt + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0199_alter_oauth2application_unique_together_and_more'), + ] + + operations = [ + migrations.RunPython(delete_clear_tokens_sjt, migrations.RunPython.noop), + migrations.AlterField( + model_name='systemjob', + name='job_type', + field=models.CharField( + blank=True, + choices=[ + ('cleanup_jobs', 'Remove jobs older than a certain number of days'), + ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), + ('cleanup_sessions', 'Removes expired browser sessions from the database'), + ], + default='', + max_length=32, + ), + ), + migrations.AlterField( + model_name='systemjobtemplate', + name='job_type', + field=models.CharField( + blank=True, + choices=[ + ('cleanup_jobs', 'Remove jobs older than a certain number of days'), + ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), + ('cleanup_sessions', 'Removes expired browser sessions from the database'), + ], + default='', + max_length=32, + ), + ), + ] diff --git a/awx/main/migrations/_create_system_jobs.py b/awx/main/migrations/_create_system_jobs.py index 14879c8aef7c..10901a809580 100644 --- a/awx/main/migrations/_create_system_jobs.py +++ b/awx/main/migrations/_create_system_jobs.py @@ -4,7 +4,7 @@ logger = logging.getLogger('awx.main.migrations') -__all__ = ['create_clearsessions_jt', 'create_cleartokens_jt'] +__all__ = ['create_clearsessions_jt', 'delete_clear_tokens_sjt'] ''' These methods are called by migrations to create various system job templates @@ -46,33 +46,8 @@ def create_clearsessions_jt(apps, schema_editor): sched.save() -def create_cleartokens_jt(apps, schema_editor): +def delete_clear_tokens_sjt(apps, schema_editor): SystemJobTemplate = apps.get_model('main', 'SystemJobTemplate') - Schedule = apps.get_model('main', 'Schedule') - ContentType = apps.get_model('contenttypes', 'ContentType') - sjt_ct = ContentType.objects.get_for_model(SystemJobTemplate) - now_dt = now() - schedule_time = now_dt.strftime('%Y%m%dT%H%M%SZ') - - sjt, created = SystemJobTemplate.objects.get_or_create( - job_type='cleanup_tokens', - defaults=dict( - name='Cleanup Expired OAuth 2 Tokens', - description='Cleanup expired OAuth 2 access and refresh tokens', - polymorphic_ctype=sjt_ct, - created=now_dt, - modified=now_dt, - ), - ) - if created: - sched = Schedule( - name='Cleanup Expired OAuth 2 Tokens', - rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1' % schedule_time, - description='Removes expired OAuth 2 access and refresh tokens', - enabled=True, - created=now_dt, - modified=now_dt, - extra_data={}, - ) - sched.unified_job_template = sjt - sched.save() + for sjt in SystemJobTemplate.objects.filter(job_type='cleanup_tokens'): + logger.info(f'Deleting system job template id={sjt.id} due to removal of local OAuth2 tokens') + sjt.delete() diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 307e49aaa6ab..e9cbfa5868a5 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -1145,7 +1145,6 @@ class SystemJobOptions(BaseModel): ('cleanup_jobs', _('Remove jobs older than a certain number of days')), ('cleanup_activitystream', _('Remove activity stream entries older than a certain number of days')), ('cleanup_sessions', _('Removes expired browser sessions from the database')), - ('cleanup_tokens', _('Removes expired OAuth 2 access tokens and refresh tokens')), ] class Meta: diff --git a/awx/main/tests/functional/test_inventory_source_migration.py b/awx/main/tests/functional/migrations/test_inventory_source_migration.py similarity index 100% rename from awx/main/tests/functional/test_inventory_source_migration.py rename to awx/main/tests/functional/migrations/test_inventory_source_migration.py diff --git a/awx/main/tests/functional/test_org_admin_migration.py b/awx/main/tests/functional/migrations/test_org_admin_migration.py similarity index 100% rename from awx/main/tests/functional/test_org_admin_migration.py rename to awx/main/tests/functional/migrations/test_org_admin_migration.py diff --git a/awx/main/tests/functional/migrations/test_rbac_migration.py b/awx/main/tests/functional/migrations/test_rbac_migration.py new file mode 100644 index 000000000000..8ee411ba1a5a --- /dev/null +++ b/awx/main/tests/functional/migrations/test_rbac_migration.py @@ -0,0 +1,49 @@ +import pytest + +from awx.main.migrations import _rbac as rbac +from awx.main.models import UnifiedJobTemplate, InventorySource, Inventory, JobTemplate, Project, Organization + + +@pytest.mark.django_db +def test_implied_organization_subquery_inventory(): + orgs = [] + for i in range(3): + orgs.append(Organization.objects.create(name='foo{}'.format(i))) + orgs.append(orgs[0]) + for i in range(4): + org = orgs[i] + if i == 2: + inventory = Inventory.objects.create(name='foo{}'.format(i)) + else: + inventory = Inventory.objects.create(name='foo{}'.format(i), organization=org) + inv_src = InventorySource.objects.create(name='foo{}'.format(i), inventory=inventory, source='ec2') + sources = UnifiedJobTemplate.objects.annotate(test_field=rbac.implicit_org_subquery(UnifiedJobTemplate, InventorySource)) + for inv_src in sources: + assert inv_src.test_field == inv_src.inventory.organization_id + + +@pytest.mark.django_db +def test_implied_organization_subquery_job_template(): + jts = [] + for i in range(5): + if i <= 3: + org = Organization.objects.create(name='foo{}'.format(i)) + else: + org = None + if i <= 4: + proj = Project.objects.create(name='foo{}'.format(i), organization=org) + else: + proj = None + jts.append(JobTemplate.objects.create(name='foo{}'.format(i), project=proj)) + # test case of sharing same org + jts[2].project.organization = jts[3].project.organization + jts[2].save() + ujts = UnifiedJobTemplate.objects.annotate(test_field=rbac.implicit_org_subquery(UnifiedJobTemplate, JobTemplate)) + for jt in ujts: + if not isinstance(jt, JobTemplate): # some are projects + assert jt.test_field is None + else: + if jt.project is None: + assert jt.test_field is None + else: + assert jt.test_field == jt.project.organization_id diff --git a/awx/main/tests/functional/migrations/test_token_sjt_removal.py b/awx/main/tests/functional/migrations/test_token_sjt_removal.py new file mode 100644 index 000000000000..35770fd4ca4e --- /dev/null +++ b/awx/main/tests/functional/migrations/test_token_sjt_removal.py @@ -0,0 +1,58 @@ +import pytest +from django.apps import apps +from django.utils.timezone import now + +from awx.main.migrations._create_system_jobs import delete_clear_tokens_sjt + +SJT_NAME = 'Cleanup Expired OAuth 2 Tokens' + + +def create_cleartokens_jt(apps, schema_editor): + # Deleted data migration + SystemJobTemplate = apps.get_model('main', 'SystemJobTemplate') + Schedule = apps.get_model('main', 'Schedule') + ContentType = apps.get_model('contenttypes', 'ContentType') + sjt_ct = ContentType.objects.get_for_model(SystemJobTemplate) + now_dt = now() + schedule_time = now_dt.strftime('%Y%m%dT%H%M%SZ') + + sjt, created = SystemJobTemplate.objects.get_or_create( + job_type='cleanup_tokens', + defaults=dict( + name=SJT_NAME, + description='Cleanup expired OAuth 2 access and refresh tokens', + polymorphic_ctype=sjt_ct, + created=now_dt, + modified=now_dt, + ), + ) + if created: + sched = Schedule( + name=SJT_NAME, + rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1' % schedule_time, + description='Removes expired OAuth 2 access and refresh tokens', + enabled=True, + created=now_dt, + modified=now_dt, + extra_data={}, + ) + sched.unified_job_template = sjt + sched.save() + + +@pytest.mark.django_db +def test_clear_token_sjt(): + SystemJobTemplate = apps.get_model('main', 'SystemJobTemplate') + Schedule = apps.get_model('main', 'Schedule') + create_cleartokens_jt(apps, None) + qs = SystemJobTemplate.objects.filter(name=SJT_NAME) + assert qs.count() == 1 + sjt = qs.first() + assert Schedule.objects.filter(unified_job_template=sjt).count() == 1 + assert Schedule.objects.filter(unified_job_template__systemjobtemplate__name=SJT_NAME).count() == 1 + + # Now run the migration logic to remove + delete_clear_tokens_sjt(apps, None) + assert SystemJobTemplate.objects.filter(name=SJT_NAME).count() == 0 + # Making sure that the schedule is cleaned up is the main point of this test + assert Schedule.objects.filter(unified_job_template__systemjobtemplate__name=SJT_NAME).count() == 0 From 3e50b019e07d166844c5edbc8ecc2fbaefc6170b Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 10 Jan 2025 13:10:24 -0500 Subject: [PATCH 2/2] Delete test file that should have been removed and fix checks (#15739) * Delete test file that should have been removed * Add more insights env variables --- .../data/inventory/plugins/insights/env.json | 4 +- .../functional/rbac/test_rbac_migration.py | 49 ------------------- 2 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 awx/main/tests/functional/rbac/test_rbac_migration.py diff --git a/awx/main/tests/data/inventory/plugins/insights/env.json b/awx/main/tests/data/inventory/plugins/insights/env.json index b87328e76187..bbece6a48fcf 100644 --- a/awx/main/tests/data/inventory/plugins/insights/env.json +++ b/awx/main/tests/data/inventory/plugins/insights/env.json @@ -1,4 +1,6 @@ { "INSIGHTS_USER": "fooo", - "INSIGHTS_PASSWORD": "fooo" + "INSIGHTS_PASSWORD": "fooo", + "INSIGHTS_CLIENT_ID": "fooo", + "INSIGHTS_CLIENT_SECRET": "fooo" } diff --git a/awx/main/tests/functional/rbac/test_rbac_migration.py b/awx/main/tests/functional/rbac/test_rbac_migration.py deleted file mode 100644 index 8ee411ba1a5a..000000000000 --- a/awx/main/tests/functional/rbac/test_rbac_migration.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest - -from awx.main.migrations import _rbac as rbac -from awx.main.models import UnifiedJobTemplate, InventorySource, Inventory, JobTemplate, Project, Organization - - -@pytest.mark.django_db -def test_implied_organization_subquery_inventory(): - orgs = [] - for i in range(3): - orgs.append(Organization.objects.create(name='foo{}'.format(i))) - orgs.append(orgs[0]) - for i in range(4): - org = orgs[i] - if i == 2: - inventory = Inventory.objects.create(name='foo{}'.format(i)) - else: - inventory = Inventory.objects.create(name='foo{}'.format(i), organization=org) - inv_src = InventorySource.objects.create(name='foo{}'.format(i), inventory=inventory, source='ec2') - sources = UnifiedJobTemplate.objects.annotate(test_field=rbac.implicit_org_subquery(UnifiedJobTemplate, InventorySource)) - for inv_src in sources: - assert inv_src.test_field == inv_src.inventory.organization_id - - -@pytest.mark.django_db -def test_implied_organization_subquery_job_template(): - jts = [] - for i in range(5): - if i <= 3: - org = Organization.objects.create(name='foo{}'.format(i)) - else: - org = None - if i <= 4: - proj = Project.objects.create(name='foo{}'.format(i), organization=org) - else: - proj = None - jts.append(JobTemplate.objects.create(name='foo{}'.format(i), project=proj)) - # test case of sharing same org - jts[2].project.organization = jts[3].project.organization - jts[2].save() - ujts = UnifiedJobTemplate.objects.annotate(test_field=rbac.implicit_org_subquery(UnifiedJobTemplate, JobTemplate)) - for jt in ujts: - if not isinstance(jt, JobTemplate): # some are projects - assert jt.test_field is None - else: - if jt.project is None: - assert jt.test_field is None - else: - assert jt.test_field == jt.project.organization_id