From 72b829a33cf583db00b38011b39d9e3c5d4cc98a Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Fri, 5 Apr 2024 11:05:20 -0400 Subject: [PATCH] Remove code that is no longer relevant There are a number of systems that no longer rely on Salt for any purpose. This removes the code pertaining to those functions in order to avoid confusion for future developers. --- pillar/apps/odlvideo.sls | 228 ---------------- pillar/apps/starcellbio.sls | 142 ---------- pillar/edx/mitx-pkgs.sls | 11 - pillar/edx/mitx-production.sls | 33 --- pillar/edx/mitx-qa.sls | 16 -- pillar/edx/mitxpro.sls | 14 - pillar/edx/scheduled_jobs.sls | 17 -- pillar/logrotate/kibana.sls | 10 - pillar/logrotate/odlvideo.sls | 10 - pillar/nginx/odlvideo.sls | 98 ------- pillar/nginx/starcellbio.sls | 56 ---- pillar/rabbitmq/devstack.sls | 37 --- pillar/rabbitmq/mitx.sls | 14 - pillar/rabbitmq/xpro.sls | 16 -- pillar/shibboleth/odlvideo.sls | 7 - pillar/top.sls | 49 ---- pillar/vault/roles/apps.sls | 115 -------- pillar/vault/roles/aws.sls | 158 ----------- pillar/vault/roles/bootcamps.sls | 29 -- pillar/vault/roles/macros.jinja | 40 --- pillar/vault/roles/micromasters.sls | 26 -- pillar/vector/odlvideo.sls | 10 - salt/apps/ocw/nextgen_build_install.sls | 122 --------- .../ocw/templates/webhook-publish.sh.jinja | 256 ------------------ salt/apps/odlvideo/configure.sls | 32 --- salt/apps/odlvideo/deploy_signal.sls | 6 - salt/apps/odlvideo/install.sls | 11 - salt/apps/odlvideo/post_deploy.sls | 15 - salt/apps/starcellbio/config.sls | 13 - salt/apps/starcellbio/install.sls | 25 -- salt/apps/starcellbio/post_deploy.sls | 26 -- salt/edx/django_user.sls | 22 -- salt/edx/edxapp_global_pre_commit.sls | 8 - salt/edx/etc_hosts.sls | 9 - salt/edx/files/edx_export_pre_commit.sh | 54 ---- salt/edx/files/mitx_devstack.yml | 47 ---- salt/edx/hacks.sls | 53 ---- salt/edx/maintenance_tasks.sls | 6 - salt/edx/migration.sls | 16 -- salt/edx/patch_nginx.sls | 21 -- salt/edx/prod.sls | 168 ------------ salt/edx/refresh_saml_provider_metadata.sls | 5 - salt/edx/templates/extra_locations_lms.j2 | 35 --- salt/edx/templates/gitreload_config.json.j2 | 11 - salt/edx/templates/gitreload_import.sh.j2 | 6 - salt/edx/templates/gitreload_init.conf.j2 | 21 -- salt/edx/templates/gitreload_site.j2 | 58 ---- salt/edx/templates/gitreload_systemd.conf.j2 | 17 -- salt/edx/templates/nginx_static_assets.j2 | 27 -- .../aws/cloud_profiles/odl-video-service.conf | 18 -- .../aws/cloud_profiles/starcellbio.conf | 13 - .../aws/odl-video/s3_bucket_policy.sls | 20 -- .../aws/odl-video/s3_buckets_cloudfront.sls | 113 -------- .../aws/s3_buckets/starcellbio.sls | 15 - .../vault/alert_cache_read_misses.sls | 12 - salt/reactors/vault/alert_expiring_leases.sls | 12 - .../vault/cache_cleanup_on_terminate.sls | 6 - salt/top.sls | 13 +- salt/vault/roles.sls | 13 - salt/vault/secret_backends.sls | 42 --- 60 files changed, 1 insertion(+), 2502 deletions(-) delete mode 100644 pillar/apps/odlvideo.sls delete mode 100644 pillar/apps/starcellbio.sls delete mode 100644 pillar/edx/mitx-pkgs.sls delete mode 100644 pillar/edx/mitx-production.sls delete mode 100644 pillar/edx/mitx-qa.sls delete mode 100644 pillar/edx/mitxpro.sls delete mode 100644 pillar/edx/scheduled_jobs.sls delete mode 100644 pillar/logrotate/kibana.sls delete mode 100644 pillar/logrotate/odlvideo.sls delete mode 100644 pillar/nginx/odlvideo.sls delete mode 100644 pillar/nginx/starcellbio.sls delete mode 100644 pillar/rabbitmq/devstack.sls delete mode 100644 pillar/rabbitmq/mitx.sls delete mode 100644 pillar/rabbitmq/xpro.sls delete mode 100644 pillar/shibboleth/odlvideo.sls delete mode 100644 pillar/vault/roles/apps.sls delete mode 100644 pillar/vault/roles/aws.sls delete mode 100644 pillar/vault/roles/bootcamps.sls delete mode 100644 pillar/vault/roles/macros.jinja delete mode 100644 pillar/vault/roles/micromasters.sls delete mode 100644 pillar/vector/odlvideo.sls delete mode 100644 salt/apps/ocw/nextgen_build_install.sls delete mode 100644 salt/apps/ocw/templates/webhook-publish.sh.jinja delete mode 100644 salt/apps/odlvideo/configure.sls delete mode 100644 salt/apps/odlvideo/deploy_signal.sls delete mode 100644 salt/apps/odlvideo/install.sls delete mode 100644 salt/apps/odlvideo/post_deploy.sls delete mode 100644 salt/apps/starcellbio/config.sls delete mode 100644 salt/apps/starcellbio/install.sls delete mode 100644 salt/apps/starcellbio/post_deploy.sls delete mode 100644 salt/edx/django_user.sls delete mode 100644 salt/edx/edxapp_global_pre_commit.sls delete mode 100644 salt/edx/etc_hosts.sls delete mode 100644 salt/edx/files/edx_export_pre_commit.sh delete mode 100644 salt/edx/files/mitx_devstack.yml delete mode 100644 salt/edx/hacks.sls delete mode 100644 salt/edx/maintenance_tasks.sls delete mode 100644 salt/edx/migration.sls delete mode 100644 salt/edx/patch_nginx.sls delete mode 100644 salt/edx/prod.sls delete mode 100644 salt/edx/refresh_saml_provider_metadata.sls delete mode 100644 salt/edx/templates/extra_locations_lms.j2 delete mode 100644 salt/edx/templates/gitreload_config.json.j2 delete mode 100644 salt/edx/templates/gitreload_import.sh.j2 delete mode 100644 salt/edx/templates/gitreload_init.conf.j2 delete mode 100644 salt/edx/templates/gitreload_site.j2 delete mode 100644 salt/edx/templates/gitreload_systemd.conf.j2 delete mode 100644 salt/edx/templates/nginx_static_assets.j2 delete mode 100644 salt/orchestrate/aws/cloud_profiles/odl-video-service.conf delete mode 100644 salt/orchestrate/aws/cloud_profiles/starcellbio.conf delete mode 100644 salt/orchestrate/aws/odl-video/s3_bucket_policy.sls delete mode 100644 salt/orchestrate/aws/odl-video/s3_buckets_cloudfront.sls delete mode 100644 salt/orchestrate/aws/s3_buckets/starcellbio.sls delete mode 100644 salt/reactors/vault/alert_cache_read_misses.sls delete mode 100644 salt/reactors/vault/alert_expiring_leases.sls delete mode 100644 salt/reactors/vault/cache_cleanup_on_terminate.sls delete mode 100644 salt/vault/roles.sls delete mode 100644 salt/vault/secret_backends.sls diff --git a/pillar/apps/odlvideo.sls b/pillar/apps/odlvideo.sls deleted file mode 100644 index 69018e0ad..000000000 --- a/pillar/apps/odlvideo.sls +++ /dev/null @@ -1,228 +0,0 @@ -# -*- mode: yaml -*- -{% set app_name = 'odl-video-service' %} -{% set python_version = '3.9.13' %} -{% set python_bin_dir = '/usr/local/pyenv/versions/{0}/bin'.format(python_version) %} -{% set ENVIRONMENT = salt.grains.get('environment', 'dev') %} -{% set env_dict = { - 'ci': { - 'env_name': 'ci', - 'bucket_suffix': 'ci', - 'domain': 'video-ci.odl.mit.edu', - 'log_level': 'DEBUG', - 'use_shibboleth': False, - 'ga_id': 'UA-5145472-26', - 'ga_view_id': '163329706', - 'transcode_pipeline_id': '1506027488410-93oya5', - 'youtube_project_id': 'ovs-youtube-qa', - 'release_branch': 'master', - 'cloudfront_subdomain': 'd2jnipcnro4zno', - 'redis_max_connections': 30, - 'EDX_BASE_URL': 'https://courses-ci.xpro.mit.edu' - }, - 'rc-apps': { - 'env_name': 'rc', - 'bucket_suffix': 'rc', - 'domain': 'video-rc.odl.mit.edu', - 'log_level': 'INFO', - 'use_shibboleth': True, - 'ga_id': 'UA-5145472-26', - 'ga_view_id': '163329706', - 'transcode_pipeline_id': '1506081628031-bepkel', - 'youtube_project_id': 'ovs-youtube-qa', - 'release_branch': 'release-candidate', - 'cloudfront_subdomain': 'du3yhovcx8dht', - 'redis_max_connections': 65000, - 'EDX_BASE_URL': 'https://courses-rc.xpro.mit.edu' - }, - 'production-apps': { - 'env_name': 'production', - 'bucket_suffix': '', - 'domain': 'video.odl.mit.edu', - 'log_level': 'WARN', - 'use_shibboleth': True, - 'ga_id': 'UA-5145472-27', - 'ga_view_id': '163330947', - 'transcode_pipeline_id': '1497541042228-8mpenl', - 'youtube_project_id': 'ovs-youtube-production', - 'release_branch': 'release', - 'cloudfront_subdomain': 'd3tsb3m56iwvoq', - 'redis_max_connections': 65000, - 'EDX_BASE_URL': 'https://courses.xpro.mit.edu' - } -} %} -{% set env_data = env_dict[ENVIRONMENT] %} -{% set minion_id = salt.grains.get('id', '') %} -{% set pg_creds = salt.vault.cached_read('postgres-{env}-odlvideo/creds/odlvideo'.format(env=ENVIRONMENT), cache_prefix=minion_id) %} -{% set rabbit_creds = salt.vault.read("secret-odl-video/rabbitmq-credentials") %} -{% set ga_json = salt.vault.read('secret-odl-video/' ~ ENVIRONMENT ~ '/ga-keyfile-json').data.value|json %} -{% set business_unit = 'odl-video' %} -{% set rds_endpoint = salt.boto_rds.get_endpoint(ENVIRONMENT ~ '-rds-postgres-odlvideo') %} -{% set redis_cluster = salt.boto3_elasticache.describe_replication_groups('ovs-{env}-redis'.format(env=env_data.env_name)) %} -{% set redis_cluster_address = redis_cluster[0].NodeGroups[0].PrimaryEndpoint.Address %} - -schedule: - refresh_{{ app_name }}_credentials: - days: 5 - function: state.sls - args: - - django.config - -python: - versions: - - number: {{ python_version }} - default: True - user: root - -django: - pip_path: {{ python_bin_dir }}/pip3 - django_admin_path: {{ python_bin_dir }}/django-admin - app_name: {{ app_name }} - settings_module: odl_video.settings - automatic_migrations: True - app_source: - type: git # Options are: git, hg, archive - revision: {{ env_data.release_branch }} - repository_url: https://github.com/mitodl/odl-video-service - state_params: - - branch: {{ env_data.release_branch }} - - force_fetch: True - - force_checkout: True - - force_reset: True - environment: - {% if env_data.env_name == 'rc' %} - FEATURE_VIDEOJS_ANNOTATIONS: True - DATABASE_URL: postgres://{{ pg_creds.data.username }}:{{ pg_creds.data.password }}@rc-apps-rds-postgres-odlvideo.cbnm7ajau6mi.us-east-1.rds.amazonaws.com/odlvideo - {% else %} - DATABASE_URL: postgres://{{ pg_creds.data.username }}:{{ pg_creds.data.password }}@{{ rds_endpoint }}/odlvideo - {% endif %} - AWS_ACCESS_KEY_ID: __vault__:cache:aws-mitx/creds/odl-video-service-{{ env_data.env_name }}>data>access_key - AWS_REGION: us-east-1 - AWS_S3_DOMAIN: s3.amazonaws.com - AWS_SECRET_ACCESS_KEY: __vault__:cache:aws-mitx/creds/odl-video-service-{{ env_data.env_name }}>data>secret_key - CLOUDFRONT_KEY_ID: __vault__::secret-operations/global/cloudfront-private-key>data>id - CLOUDFRONT_PRIVATE_KEY: __vault__::secret-operations/global/cloudfront-private-key>data>value - CELERY_BROKER_URL: redis://{{ redis_cluster_address }}:6379/1 - DJANGO_LOG_LEVEL: {{ env_data.log_level }} - DROPBOX_FOLDER: /Captions - DROPBOX_KEY: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/dropbox_app>data>key - DROPBOX_TOKEN: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/dropbox_app>data>token - EDX_BASE_URL: {{ env_data.EDX_BASE_URL }} - ENABLE_VIDEO_PERMISSIONS: True - FIELD_ENCRYPTION_KEY: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/field-encryption-key>data>value - ET_PIPELINE_ID: {{ env_data.transcode_pipeline_id }} - ET_PRESET_IDS: 1504127981921-c2jlwt,1504127981867-06dkm6,1504127981819-v44xlx,1504127981769-6cnqhq,1351620000001-200040,1351620000001-200050 - FEATURE_RETRANSCODE_ENABLED: True - GA_DIMENSION_CAMERA: dimension1 - GA_KEYFILE_JSON: '{{ ga_json }}' - GA_VIEW_ID: {{ env_data.ga_view_id }} - GA_TRACKING_ID: {{ env_data.ga_id }} - LECTURE_CAPTURE_USER: {{ salt.sdb.get('sdb://consul/odl-video-service/lecture-capture-user') }} - MAILGUN_KEY: __vault__::secret-operations/global/mailgun-api-key>data>value - MAILGUN_URL: https://api.mailgun.net/v3/video-mail.odl.mit.edu - # Cert and private key need to be stored in vault as strings - MIT_WS_CERTIFICATE: __vault__::secret-{{ business_unit }}/{{ env_data.env_name }}/mit-application-certificate>data>certificate - MIT_WS_PRIVATE_KEY: __vault__::secret-{{ business_unit }}/{{ env_data.env_name }}/mit-application-certificate>data>private_key - ODL_VIDEO_ADMIN_EMAIL: cuddle_bunnies@mit.edu - ODL_VIDEO_BASE_URL: https://{{ env_data.domain }} - ODL_VIDEO_ENVIRONMENT: {{ ENVIRONMENT }} - ODL_VIDEO_FROM_EMAIL: MIT ODL Video - ODL_VIDEO_LOG_LEVEL: {{ env_data.log_level }} - ODL_VIDEO_SUPPORT_EMAIL: MIT ODL Video - ODL_VIDEO_LOG_FILE: /var/log/odl-video/django.log - OPENEDX_API_CLIENT_ID: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/openedx-api>data>client_id - OPENEDX_API_CLIENT_SECRET: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/openedx-api>data>client_secret - REDIS_URL: redis://{{ redis_cluster_address }}:6379/0 - REDIS_MAX_CONNECTIONS: {{ env_data.redis_max_connections }} - SECRET_KEY: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/django-secret-key>data>value - SENTRY_DSN: __vault__::secret-{{ business_unit }}/global/sentry-dsn>data>value - STATUS_TOKEN: {{ ENVIRONMENT }} - USE_SHIBBOLETH: {{ env_data.use_shibboleth }} - VIDEO_CLOUDFRONT_DIST: {{ env_data.cloudfront_subdomain }} - VIDEO_S3_BUCKET: odl-video-service{{ '-{}'.format(env_data.bucket_suffix).rstrip('-') }} - VIDEO_S3_SUBTITLE_BUCKET: odl-video-service-subtitles{{ '-{}'.format(env_data.bucket_suffix).rstrip('-') }} - VIDEO_S3_THUMBNAIL_BUCKET: odl-video-service-thumbnails{{ '-{}'.format(env_data.bucket_suffix).rstrip('-') }} - VIDEO_S3_TRANSCODE_BUCKET: odl-video-service-transcoded{{ '-{}'.format(env_data.bucket_suffix).rstrip('-') }} - VIDEO_S3_WATCH_BUCKET: odl-video-service-uploaded{{ '-{}'.format(env_data.bucket_suffix).rstrip('-') }} - VIDEO_STATUS_UPDATE_FREQUENCY: 60 - VIDEO_WATCH_BUCKET_FREQUENCY: 600 - YT_ACCESS_TOKEN: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/youtube-credentials>data>access_token - YT_CLIENT_ID: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/youtube-credentials>data>client_id - YT_CLIENT_SECRET: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/youtube-credentials>data>client_secret - YT_PROJECT_ID: {{ env_data.youtube_project_id }} - YT_REFRESH_TOKEN: __vault__::secret-{{ business_unit }}/{{ ENVIRONMENT }}/youtube-credentials>data>refresh_token - pkgs: - - git - - build-essential - - libssl-dev - - libjpeg-dev - - zlib1g-dev - - libpqxx-dev - - libxml2-dev - - libffi-dev - - libmariadbclient-dev - states: - setup: - - apps.odlvideo.install - config: - - apps.odlvideo.configure - post_install: - - apps.odlvideo.post_deploy - - apps.odlvideo.deploy_signal - -uwsgi: - overrides: - pip_path: {{ python_bin_dir }}/pip3 - uwsgi_path: {{ python_bin_dir }}/uwsgi - emperor_config: - uwsgi: - - logto: /var/log/uwsgi/emperor.log - apps: - {{ app_name }}: - uwsgi: - - strict: 'true' - - enable-threads: 'true' - - vacuum: 'true' - - single-interpreter: 'true' - - die-on-term: 'true' - - need-app: 'true' - - disable-logging: 'true' - - log-4xx: 'true' - - log-5xx: 'true' - - max-requests: '1000' - - max-worker-lifetime: '3600' - - reload-on-rss: '200' - - worker-reload-mercy: '60' - - harakiri: '60' - - py-call-osafterfork: 'true' - - buffer-size: '65535' - - post-buffering: '65535' - - auto-procname: 'true' - - chdir: /opt/{{ app_name }} - - chown-socket: 'www-data:deploy' - - disable-write-exception: 'true' - - gid: deploy - - logto: /var/log/uwsgi/apps/%n.log - - memory-report: 'true' - - module: odl_video.wsgi - - pidfile: /var/run/uwsgi/{{ app_name }}.pid - - processes: 2 - - pyhome: /usr/local/pyenv/versions/{{ python_version }}/ - - socket: /var/run/uwsgi/{{ app_name }}.sock - - threads: 50 - - thunder-lock: 'true' - - touch-reload: /opt/{{ app_name }}/deploy_complete.txt - - uid: deploy - - attach-daemon2: >- - cmd=/usr/local/pyenv/versions/{{ python_version }}/bin/celery worker -A odl_video -B --pidfile /opt/{{ app_name }}/celery.pid -l {{ env_data.log_level }}, - pidfile=/opt/{{ app_name }}/celery.pid, - daemonize=true, - touch=/opt/{{ app_name}}/deploy_complete.txt - - -node: - install_from_binary: True - version: 13.13.0 - -beacons: - memusage: - - percent: 95% diff --git a/pillar/apps/starcellbio.sls b/pillar/apps/starcellbio.sls deleted file mode 100644 index 91ae7977b..000000000 --- a/pillar/apps/starcellbio.sls +++ /dev/null @@ -1,142 +0,0 @@ -# -*- mode: yaml -*- -{% set app_name = 'starcellbio' %} -{% set python_version = '2.7.15' %} -{% set python_bin_dir = '/usr/local/pyenv/versions/{0}/bin'.format(python_version) %} -{% set ENVIRONMENT = salt.grains.get('environment', 'rc-apps') %} -{% set minion_id = salt.grains.get('id', '') %} -{% set env_dict = { - 'rc-apps': { - 'log_level': 'DEBUG', - 'release_branch': 'develop' - }, - 'production-apps': { - 'log_level': 'WARN', - 'release_branch': 'release' - } -} %} -{% set env_data = env_dict[ENVIRONMENT] %} -{% set rds_endpoint = salt.boto_rds.get_endpoint(ENVIRONMENT ~ '-rds-mariadb-starcellbio') %} - -schedule: - refresh_{{ app_name }}_credentials: - days: 5 - function: state.sls - args: - - django.config - -python: - versions: - - number: {{ python_version }} - default: True - user: root - -django: - pip_path: {{ python_bin_dir }}/pip2 - django_admin_path: {{ python_bin_dir }}/django-admin - app_name: {{ app_name }} - settings_module: StarCellBio.settings - automatic_migrations: True - app_source: - type: git # Options are: git, hg, archive - revision: {{ env_data.release_branch }} - repository_url: https://github.com/starteam/starcellbio_html.git - state_params: - - branch: {{ env_data.release_branch }} - - force_fetch: True - - force_checkout: True - - force_reset: True - - user: deploy - pkgs: - - build-essential - - git - - libjpeg-dev - - libmariadbclient-dev - - libncurses5-dev - - libssl-dev - - libxml2-dev - - libxslt1-dev - - mariadb-client - - nodejs - - openjdk-8-jre - - python-mysqldb - - sendmail - - zlib1g-dev - states: - setup: - - apps.starcellbio.install - config: - - apps.starcellbio.config - post_install: - - apps.starcellbio.post_deploy - -uwsgi: - overrides: - pip_path: {{ python_bin_dir }}/pip - uwsgi_path: {{ python_bin_dir }}/uwsgi - emperor_config: - uwsgi: - - logto: /var/log/uwsgi/emperor.log - apps: - {{ app_name }}: - uwsgi: - - strict: 'true' - - enable-threads: 'true' - - vacuum: 'true' - - single-interpreter: 'true' - - die-on-term: 'true' - - need-app: 'true' - - disable-logging: 'true' - - log-4xx: 'true' - - log-5xx: 'true' - - max-requests: '1000' - - max-worker-lifetime: '3600' - - reload-on-rss: '200' - - worker-reload-mercy: '60' - - harakiri: '60' - - py-call-osafterfork: 'true' - - buffer-size: '65535' - - post-buffering: '65535' - - auto-procname: 'true' - - chdir: /opt/{{ app_name }} - - chown-socket: 'www-data:deploy' - - disable-write-exception: 'true' - - gid: deploy - - logto: /var/log/uwsgi/apps/%n.log - - memory-report: 'true' - - module: StarCellBio.wsgi - - pidfile: /var/run/uwsgi/{{ app_name }}.pid - - processes: 2 - - pyhome: /usr/local/pyenv/versions/{{ python_version }}/ - - socket: /var/run/uwsgi/{{ app_name }}.sock - - threads: 50 - - thunder-lock: 'true' - - uid: deploy - -starcellbio: - config: - SECRET_KEY: __vault__:gen_if_missing:secret-starteam/{{ ENVIRONMENT }}/starcellbio/django-secret-key>data>value - PROJECT_HOME: /opt/{{ app_name }} - DEBUG: False - LOG_LEVEL: {{ env_data.log_level }} - SCB_TIME_ZONE: America/New_York - DB_ENGINE: django.db.backends.mysql - DB_NAME: starcellbio - DB_USER: __vault__:cache:mariadb-{{ ENVIRONMENT }}-starcellbio/creds/starcellbio>data>username - DB_PASSWORD: __vault__:cache:mariadb-{{ ENVIRONMENT }}-starcellbio/creds/starcellbio>data>password - DB_HOST: {{ rds_endpoint.split(":")[0] }} - DB_PORT: 3306 - TEMPLATE_DEBUG: false - SERVER_EMAIL: mitxmail@mit.edu - ADMINS: - - - - mitx-devops - - mitx-devops@mit.edu - S3_BACKEND_ENABLED: True - DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage - AWS_ACCESS_KEY_ID: __vault__:cache:aws-mitx/creds/read-write-delete-scb-{{ ENVIRONMENT }}-microscopy-uploads>data>access_key - AWS_SECRET_ACCESS_KEY: __vault__:cache:aws-mitx/creds/read-write-delete-scb-{{ ENVIRONMENT }}-microscopy-uploads>data>secret_key - AWS_STORAGE_BUCKET_NAME: scb-{{ ENVIRONMENT }}-microscopy-uploads - -node: - install_from_binary: True - version: 8.11.4 diff --git a/pillar/edx/mitx-pkgs.sls b/pillar/edx/mitx-pkgs.sls deleted file mode 100644 index b5e4b0ede..000000000 --- a/pillar/edx/mitx-pkgs.sls +++ /dev/null @@ -1,11 +0,0 @@ -edx: - dependencies: - os_packages: - - git - - mysql-client - - libmysqlclient-dev - - landscape-common - - libssl-dev - - virtualenv - - nfs-common - - postfix diff --git a/pillar/edx/mitx-production.sls b/pillar/edx/mitx-production.sls deleted file mode 100644 index 91705b039..000000000 --- a/pillar/edx/mitx-production.sls +++ /dev/null @@ -1,33 +0,0 @@ -#!jinja|yaml|gpg - -edx: - tracking_backups: - aws_creds: - access_key: __vault__:cache:aws-mitx/creds/read-write-odl-residential-tracking-backup>data>access_key - secret_key: __vault__:cache:aws-mitx/creds/read-write-odl-residential-tracking-backup>data>secret_key - gitreload: - basic_auth: - username: mitx - password: __vault__::secret-residential/mitx-production/gitreload>data>value - edxapp: - max_upload_size: 50 - dependencies: - os_packages: - - git - - libmysqlclient-dev - - mariadb-client-10.3 - - landscape-common - - libssl-dev - - python3-pip - - python3-virtualenv - - nfs-common - - postfix - -users: - ichuang: - - 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCDegFnWcIrLwQLUlSEhfY1KZyJAt//Uzn3k5RUSBC/E8kzznxnMPPUN+D42Gwf/Y3aZxplL7WMqE6hu7L7ANsSVnkCBy63ZsUmA0p4owiYlad8NDlhNxYfEEtmqDDE1g0Uqv5X+1HkcKOWxvqjGWVUzxndSQZRaAgtjzWVmOdWrIpNFM3iEO8mKe3d9wTg+iEk4TPVq7U22dLwwtBT0axjzdDVvyEu0zl8diyUJNZmzp+AK+Q56LAMi72/pnBvxV4kYBvLNvxaGc+wEigv10v3EGqWPVA10rDulBjzF4DHOQogCOBxEDPDhQXnyYRTRmclcwB262HA/4JlyslxkGw7 ocw-chuang' - - 'ssh-dss AAAAB3NzaC1kc3MAAACBAI70pqMzB9oN23EL51ByF0/GYxNI3R9G6IV1rjOPls7FzWNITmlAm0UEyHRR7P+jveCqFRd1uj+Fb05yBXUmL+cVQl1JnG3G6fwChop9q18yynngZrD3balzVe0x27gnhMUdqPjaDo3SufwqM3pWVjYY9PdNuzxwzh/MrewuZivPAAAAFQDAA2qMBUcHsu1r0lW356ncf0V0mQAAAIBiqw6ENw3q9VXQv7x4MHs5G1xqUV8sUOugD+ZoZ5nzZJtgAJrjX4XnVO+1xxjVbCSbpiw8Jyxu0Zl3akeI/x9+j6qKE3zKPfNgtBJhITZLA01w5Z1wMvJ51z6gWvyXLDxmePC1AM/9ZyhYewWztGy94mePFmRuCauURQTUIJZL6wAAAIAzbih1fNn7lS/y5Agt7lxf4My1B/TGbFRpsoSV35Dvt7zIlwHUdG8FcrqRIamFV3c3Gb42zcDruDvuinrtIIgXXGEQ167SUALoID1e7MBiK4SUWIrhYLC7BpcFl3rM21cwJawPgFtsMVPn7pzOUkUXsMWED/u3/OtI7rmDnSHMDA== ike@f' - - 'ssh-dss AAAAB3NzaC1kc3MAAACBANr+McQzQGhseUuBGjjgNKiWCVybt9074lgZhkg4RJ1YVEwzBairR4r9jFPZQ++rS/FqOADloTyfG4wzaQoszefZv/KpFu8vn1HFi3ZvEn699syBbcM8RbSsR2RhtosqcyAFi/qvJyqL54WgR52VBJ4y55kma1FE7LrHgQ5lDP4xAAAAFQDowE/0SswsfGJ9IDnl2pj4U4r87QAAAIEA1E8KW08oSyVibmcZVyGibqrlkMb1Feyrl79icvr1MgPserBYliqN+qkMCxTqobHBAYARLqLrDKte7K8HmCa98Ri06XRYUNF/lRKOOfoupFV4H4xP5dTNrNu2nqU0Rk/EHmsU1492nbNcdifY4cj8YC64htI+hu+bYKH7gsGiFNUAAACALZCPQOvKLOpHOXgRJQHLlLA+UDiVHzL/bcKtaDw8pTerU9jUIeOSF03SszyzViW52x8x8q0rr18IrqaRLB14LeV6wo5kRu4JYgp3JrizA4Bt9YVm+7v9s5dgL8hk9Bkju4ZqskymuXCaRatT+pDUbhoeNi0qSPOofH0lNQ6mtNI= ike@te' - - 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxNgkS88ONPJiHPF9CkgD06NjHoXznX63XX+rGFUs+X1EbPChqA+41ysr64goulhEamwzDOCZhWm+mdcks3ZT7drc523BrjSH/oVE8MZCTBa3k1z/l1G9iGqeDqsYkcD2DTPFRuXLZLVjfJSE3eaXNFFUGWXHQcOfupGrz2nNBdAreRBbjRy0ZO9geWAP9l9b7QPEncY1rctd8cyrSsP2Iz/zkt7bxV7kcdQb2BpN7HBICfmW8TwTC68pvLzBsZHd6QLqOHySJ+LgAhs17eBGooDiX1i6Sk6uKgOJtA/8v+Z6EAb1dk5LNT/6P5y8VmLjHCbc/KnAmllfq6sajBeKFQ== ike@tea' - - 'ssh-dss AAAAB3NzaC1kc3MAAACBAM9M8k5XDx88567Y2jxlg/zRcHUVHk42v9xWLBJY4G4VwBmhiWtQzfqX/+PuW7F5ipLR3WHmcsSBSKYN6V2QNKisURLm+PHbOVfbcJ8WfCQDxJgwTSuVdu1terltqYNPj9OOAsm9jz+cn58FOh0lM4Mv2GGsoMM9rllLpXyFi77nAAAAFQCyEIIyfbbkDUekiong0fFlG+Y1TQAAAIBYbHVio+mNdCjL3DkwJyHRvHqlFIKbFfCbkUBqM62hnipjWytOC0P9AdH3cZwYRCo1AAh3QAI0p/hTTUHYyG6WyUcXMx5KaZqgxq+l3Be5xMzEoIj8nCU2XJNko5A2ia6WkEp700U/GHKQyyz6bj5tueQusT1NofaP3O/Y8ZAaYAAAAIEAvhea37T06IcTlCEzKdQ5qiObYJgNwVqroCSJ3DV5SMNZkYSLiqSXADg4eWKYILivOqevSxCTwv+fWOiPwZ4MKkHtCAksQ9VPEspONm6nSrsgnC11m1DrVZVx3i/4bI71sp3wIz6HhsC0R9dTKcPXDu2RV3x18Bpf7YxggZer5kc= ike@ikes' - - 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArqmhtieZKYvRqzHvNVCzvQXbXqVXJXuARb8PXkgCsBrDKpxfVjYhd6Uwt+4K/OAgUpPRABBg5uKdFPDEIFxEMZwZfZfj2Tk3wFtDHs/z1ravwdUVrR+0WBP1Gqn/rmfrsqHRKNwaHD+DHeIWI7FvRQxHV8s1YZgzzZVC0hHKh24bSwynDtVBstT7uiQ+oZMyqtyYNH+tnRBNdlB/2xeK4grA7J7aCR5k6agfVG7WlMCTKIZr4ej3WUfCffjstz8CcBVNPb1dfTN8oHC9kgRj0BCeiLH8Xj1o1rFO58P+HiuvGcKbZzRgymNe5EnoY3OpoP6s95te3Ll73AEIJEX+bQ== ike@route' diff --git a/pillar/edx/mitx-qa.sls b/pillar/edx/mitx-qa.sls deleted file mode 100644 index 3486c9e81..000000000 --- a/pillar/edx/mitx-qa.sls +++ /dev/null @@ -1,16 +0,0 @@ -edx: - gitreload: - basic_auth: - username: mitx - password: __vault__::secret-residential/mitx-qa/gitreload>data>value - dependencies: - os_packages: - - git - - libmysqlclient-dev - - mariadb-client-10.3 - - landscape-common - - libssl-dev - - python3-pip - - python3-virtualenv - - nfs-common - - postfix diff --git a/pillar/edx/mitxpro.sls b/pillar/edx/mitxpro.sls deleted file mode 100644 index dcdbc8a14..000000000 --- a/pillar/edx/mitxpro.sls +++ /dev/null @@ -1,14 +0,0 @@ -{% set business_unit = salt.grains.get('business_unit') %} -{% set environment = salt.grains.get('environment') %} - -edx: - mitxpro: - registration_access_token: __vault__:gen_if_missing:secret-{{ business_unit }}/{{ environment }}/xpro-registration-access-token>data>value - tls_key: __vault__::secret-operations/global/xpro_wildcard_cert>data>key - tls_crt: __vault__::secret-operations/global/xpro_wildcard_cert>data>cert -{% if environment == 'mitxpro-production' %} - tracking_backups: - aws_creds: - access_key: __vault__:cache:aws-mitx/creds/read-write-odl-mitxpro-tracking-backup>data>access_key - secret_key: __vault__:cache:aws-mitx/creds/read-write-odl-mitxpro-tracking-backup>data>secret_key -{% endif %} diff --git a/pillar/edx/scheduled_jobs.sls b/pillar/edx/scheduled_jobs.sls deleted file mode 100644 index 0947c4e66..000000000 --- a/pillar/edx/scheduled_jobs.sls +++ /dev/null @@ -1,17 +0,0 @@ -schedule: - delete_edx_logs_older_than_30_days: - maxrunning: 1 - when: Sunday 5:00am - function: state.sls - args: - - edx.maintenance_tasks - {% if 'edx-worker' in salt.grains.get('roles') %} - restart_edx_worker_services: - days: 5 - splay: 30 - function: supervisord.restart - args: - - all - kwargs: - bin_env: /edx/bin/supervisorctl - {% endif %} \ No newline at end of file diff --git a/pillar/logrotate/kibana.sls b/pillar/logrotate/kibana.sls deleted file mode 100644 index 6ee3930b0..000000000 --- a/pillar/logrotate/kibana.sls +++ /dev/null @@ -1,10 +0,0 @@ -logrotate: - kibana: - name: /var/log/kibana.log - options: - - rotate 4 - - weekly - - copytruncate - - notifempty - - compress - - delaycompress diff --git a/pillar/logrotate/odlvideo.sls b/pillar/logrotate/odlvideo.sls deleted file mode 100644 index a9a1e6ed3..000000000 --- a/pillar/logrotate/odlvideo.sls +++ /dev/null @@ -1,10 +0,0 @@ -logrotate: - odlvideo: - name: /var/log/odl-video/django.log - options: - - rotate 7 - - daily - - copytruncate - - notifempty - - compress - - delaycompress diff --git a/pillar/nginx/odlvideo.sls b/pillar/nginx/odlvideo.sls deleted file mode 100644 index 181154c94..000000000 --- a/pillar/nginx/odlvideo.sls +++ /dev/null @@ -1,98 +0,0 @@ -{% set app_name = 'odl-video-service' %} -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} -{% set ENVIRONMENT = salt.grains.get('environment', 'rc-apps') %} -{% set env_data = env_settings.environments[ENVIRONMENT] %} -{% set server_domain_names = env_data.purposes['odl-video-service'].domains %} -{% set ovs_login_path = 'login' %} - -nginx: - install_from_source: True - source_version: 1.13.8 - source_hash: 8410b6c31ff59a763abf7e5a5316e7629f5a5033c95a3a0ebde727f9ec8464c5 - certificates: - odl_wildcard: - public_cert: __vault__::secret-operations/global/ovs_web_cert>data>value - private_key: __vault__::secret-operations/global/ovs_web_cert>data>key - server: - extra_config: - shib_params: - source_path: salt://nginx/files/default/nginx.conf - shib_request_set: - - $shib_remote_user $upstream_http_variable_remote_user - - $shib_eppn $upstream_http_variable_eppn - - $shib_mail $upstream_http_variable_mail - - $shib_displayname $upstream_http_variable_displayname - uwsgi_param: - - REMOTE_USER $shib_remote_user - - EPPN $shib_eppn - - MAIL $shib_mail - - DISPLAY_NAME $shib_displayname - servers: - managed: - {{ app_name }}: - enabled: True - config: - - server: - - server_name: {{ server_domain_names|tojson }} - - listen: 80 - - listen: '[::]:80' - - location /: - - return: 301 https://$host$request_uri - - server: - - server_name: {{ server_domain_names|tojson }} - - listen: '443 ssl default_server' - - listen: '[::]:443 ssl' - - root: /opt/odl-video-service/ - - ssl_certificate: /etc/nginx/ssl/odl_wildcard.crt - - ssl_certificate_key: /etc/nginx/ssl/odl_wildcard.key - - ssl_stapling: 'on' - - ssl_stapling_verify: 'on' - - ssl_session_timeout: 1d - - ssl_session_tickets: 'off' - - ssl_protocols: 'TLSv1 TLSv1.1 TLSv1.2 TLSv1.3' - - ssl_ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256\ - :DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384\ - :ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256\ - :ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256" - - ssl_prefer_server_ciphers: 'on' - - resolver: 1.1.1.1 - - location /shibauthorizer: - - internal: '' - - include: fastcgi_params - - include: includes/shib_fastcgi_params - - fastcgi_pass: 'unix:/run/shibauthorizer.sock' - - location /Shibboleth.sso: - - include: fastcgi_params - - include: includes/shib_fastcgi_params - - fastcgi_pass: 'unix:/run/shibresponder.sock' - - location /{{ ovs_login_path }}: - - include: includes/shib_clear_headers - - shib_request: /shibauthorizer - - shib_request_use_headers: 'on' - - include: conf.d/shib_params.conf - - include: uwsgi_params - - uwsgi_ignore_client_abort: 'on' - - uwsgi_pass: unix:/var/run/uwsgi/odl-video-service.sock - - location /status: - - include: uwsgi_params - - uwsgi_pass: unix:/var/run/uwsgi/odl-video-service.sock - - location /: - - include: uwsgi_params - - uwsgi_ignore_client_abort: 'on' - - uwsgi_pass: unix:/var/run/uwsgi/odl-video-service.sock - - location ~* /static/(.*$): - - expires: max - - add_header: 'Access-Control-Allow-Origin *' - - try_files: '$uri $uri/ /staticfiles/$1 /staticfiles/$1/ =404' - - location /collections/letterlocking: - - return: 301 https://www.youtube.com/c/Letterlocking/videos - - location /collections/letterlocking/videos: - - return: 301 https://www.youtube.com/c/Letterlocking/videos - - location /collections/letterlocking/videos/30213-iron-gall-ink-a-quick-and-easy-method: - - return: 301 https://www.youtube.com/playlist?list=PL2uZTM-xaHP4tFQT7eTTK3sWRoJMcDWwB - - location /collections/letterlocking/videos/30215-elizabeth-stuart-s-deciphering-sir-thomas-roe-s-letter-cryptography-1626: - - return: 301 https://www.youtube.com/watch?v=6X_ZXrLs8I8&list=PL2uZTM-xaHP4tFQT7eTTK3sWRoJMcDWwB&index=3&t=0s - - location /collections/letterlocking/videos/30209-a-tiny-spy-letter-constantijn-huygens-to-amalia-von-solms-1635: - - return: 301 https://www.youtube.com/watch?v=PePWd-h679c&list=PL2uZTM-xaHP4tFQT7eTTK3sWRoJMcDWwB&index=7&t=0s - - location /collections/c8c5179c7596408fa0f09f6b76082331: - - return: 301 https://www.youtube.com/c/MITEnergyInitiative diff --git a/pillar/nginx/starcellbio.sls b/pillar/nginx/starcellbio.sls deleted file mode 100644 index 9b42b73fb..000000000 --- a/pillar/nginx/starcellbio.sls +++ /dev/null @@ -1,56 +0,0 @@ -{% set app_name = 'starcellbio' %} -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} -{% set ENVIRONMENT = salt.grains.get('environment', 'rc-apps') %} -{% set env_data = env_settings.environments[ENVIRONMENT] %} -{% set server_domain_names = env_data.purposes[app_name].domains %} -{% set ovs_login_path = 'login' %} - -nginx: - install_from_ppa: True - certificates: - starcellbio: - public_cert: __vault__::secret-starteam/global/starcellbio/ssl>data>cert - private_key: __vault__::secret-starteam/global/starcellbio/ssl>data>key - servers: - managed: - {{ app_name }}: - enabled: True - config: - - server: - - server_name: '{{ server_domain_names|join(' ') }} ""' - - listen: 80 - - listen: '[::]:80' - - location /status: - - return: 200 OK - - add_header: Content-Type text/plain - - location /: - - return: 301 https://$host$request_uri - - server: - - server_name: '{{ server_domain_names|join(' ') }} ""' - - listen: '443 ssl default_server' - - listen: '[::]:443 ssl' - - root: /opt/{{ app_name }}/ - - ssl_certificate: /etc/nginx/ssl/starcellbio.crt - - ssl_certificate_key: /etc/nginx/ssl/starcellbio.key - - ssl_stapling: 'on' - - ssl_stapling_verify: 'on' - - ssl_session_timeout: 1d - - ssl_session_tickets: 'off' - - ssl_protocols: 'TLSv1 TLSv1.1 TLSv1.2 TLSv1.3' - - ssl_ciphers: "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256\ - :DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384\ - :ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256\ - :ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256" - - ssl_prefer_server_ciphers: 'on' - - resolver: 1.1.1.1 - - location /status: - - return: 200 OK - - add_header: Content-Type text/plain - - location /: - - include: uwsgi_params - - uwsgi_ignore_client_abort: 'on' - - uwsgi_pass: unix:/var/run/uwsgi/{{ app_name }}.sock - - location ~* /static/(.*$): - - expires: max - - add_header: 'Access-Control-Allow-Origin *' - - try_files: '$uri $uri/ /staticfiles/$1 /staticfiles/$1/ =404' diff --git a/pillar/rabbitmq/devstack.sls b/pillar/rabbitmq/devstack.sls deleted file mode 100644 index 3bd73e21b..000000000 --- a/pillar/rabbitmq/devstack.sls +++ /dev/null @@ -1,37 +0,0 @@ -#!jinja|yaml - -{% set rabbitmq_admin_password = 'changeme' %} -{% set ENVIRONMENT = salt.grains.get('environment') %} -{% set BUSINESS_UNIT = salt.grains.get('business_unit', 'residential') %} - -rabbitmq: - overrides: - version: '3.7.4-1' - erlang_version: '1:20.1' - configuration: - disk_free_limit.relative: 0.2 - auth_backends.1: rabbit_auth_backend_internal - users: - - name: guest - state: absent - - name: admin - state: present - settings: - tags: - - administrator - perms: - - '/xqueue': - - '.*' - - '.*' - - '.*' - - '/celery': - - '.*' - - '.*' - - '.*' - password: {{ rabbitmq_admin_password }} - vhosts: - - name: '/xqueue' - state: present - - name: '/celery' - state: present - erlang_cookie: __vault__:gen_if_missing:secret-{{ BUSINESS_UNIT }}/{{ ENVIRONMENT }}/erlang_cookie>data>value diff --git a/pillar/rabbitmq/mitx.sls b/pillar/rabbitmq/mitx.sls deleted file mode 100644 index faa8c00a8..000000000 --- a/pillar/rabbitmq/mitx.sls +++ /dev/null @@ -1,14 +0,0 @@ -#!jinja|yaml|gpg - -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} -{% set ENVIRONMENT = salt.grains.get('environment') %} -{% set BUSINESS_UNIT = salt.grains.get('business_unit', 'residential') %} - -rabbitmq: - vhosts: - {% for purpose in env_settings['environments'][ENVIRONMENT].purposes %} - - name: /xqueue_{{ purpose|replace('-', '_') }} - state: present - - name: /celery_{{ purpose|replace('-', '_') }} - state: present - {% endfor %} diff --git a/pillar/rabbitmq/xpro.sls b/pillar/rabbitmq/xpro.sls deleted file mode 100644 index e01db78a4..000000000 --- a/pillar/rabbitmq/xpro.sls +++ /dev/null @@ -1,16 +0,0 @@ -#!jinja|yaml|gpg - -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} -{% set ENVIRONMENT = salt.grains.get('environment') %} -{% set BUSINESS_UNIT = salt.grains.get('business_unit', 'residential') %} - -rabbitmq: - vhosts: - {% for purpose in env_settings['environments'][ENVIRONMENT].purposes %} - - name: /xqueue_{{ purpose|replace('-', '_') }} - state: present - - name: /celery_{{ purpose|replace('-', '_') }} - state: present - - name: /video_{{ purpose|replace('-', '_') }} - state: present - {% endfor %} diff --git a/pillar/shibboleth/odlvideo.sls b/pillar/shibboleth/odlvideo.sls deleted file mode 100644 index 09c5aca5f..000000000 --- a/pillar/shibboleth/odlvideo.sls +++ /dev/null @@ -1,7 +0,0 @@ -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} -{% set ENVIRONMENT = salt.grains.get('environment', 'rc-apps') %} - -nginx-shibboleth: - secrets: - key: __vault__::secret-odl-video/{{ ENVIRONMENT }}/shibboleth/sp-key>data>value - cert: __vault__::secret-odl-video/{{ ENVIRONMENT }}/shibboleth/sp-cert>data>value diff --git a/pillar/top.sls b/pillar/top.sls index 3838712af..f7003cedd 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -16,41 +16,6 @@ base: - master.production_schedule master-operations-qa: - master.qa_schedule - starcellbio*: - - apps.starcellbio - - nginx - - nginx.starcellbio - - consul - odl-video-service*: - - apps.odlvideo - - nginx - - nginx.odlvideo - - consul - - shibboleth - - shibboleth.odlvideo - - logrotate.odlvideo - - vector.odlvideo - 'proxy-bootcamps-*': - - match: glob - - heroku.bootcamps - 'proxy-micromasters-*': - - match: glob - - heroku.micromasters - 'proxy-mitxpro-*': - - match: glob - - heroku.xpro - 'proxy-mit-open-discussions-*': - - match: glob - - heroku.discussions - 'proxy-mitopen-*': - - match: glob - - heroku.mitopen - 'proxy-mitxonline-*': - - match: glob - - heroku.mitxonline - 'proxy-ocw-studio-*': - - match: glob - - heroku.ocw-studio 'roles:mitx-cas': - match: grain - apps.mitx_cas @@ -138,17 +103,3 @@ base: 'roles:ocw-db': - match: grain - logrotate.ocw_cms - 'G@roles:ocw-build and G@environment:production-apps': - - match: compound - - apps.ocw-next-production - - caddy - - caddy.ocw_build - 'G@roles:ocw-build and G@environment:rc-apps': - - match: compound - - apps.ocw-next-qa - - caddy - - caddy.ocw_build - 'roles:ocw-build': - - match: grain - - logrotate.ocw_build - - vector.ocw_build diff --git a/pillar/vault/roles/apps.sls b/pillar/vault/roles/apps.sls deleted file mode 100644 index 85291ba84..000000000 --- a/pillar/vault/roles/apps.sls +++ /dev/null @@ -1,115 +0,0 @@ -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} -{% set SIX_MONTHS = '4368h' %} -vault: - roles: - {% for env in ['rc-apps', 'production-apps'] %} - {% for app in ['reddit', 'odlvideo'] %} - rabbitmq-{{ env }}-{{ app }}: - backend: rabbitmq-{{ env }} - name: {{ app }} - options: - vhosts: '{"/{{ app }}": {"write": ".*", "read": ".*", "configure": ".*"}}' - postgresql_{{ env }}_{{ app }}_admin: - backend: postgres-{{ env }}-{{ app }} - name: admin - options: - db_name: {{ app }} - default_ttl: {{ SIX_MONTHS }} - max_ttl: {{ SIX_MONTHS }} - creation_statements: >- - {% raw %}CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'{% endraw %} IN ROLE "rds_superuser" INHERIT CREATEROLE CREATEDB; - GRANT "{{app}}" TO {% raw %}"{{name}}"{% endraw %} WITH ADMIN OPTION; - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO {% raw %}"{{name}}"{% endraw %} WITH GRANT OPTION; - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO {% raw %}"{{name}}"{% endraw %} WITH GRANT OPTION; - {% raw %} - revocation_statements: >- - GRANT "{{name}}" TO odldevops WITH ADMIN OPTION; - REASSIGN OWNED BY "{{name}}" TO {% endraw %}"{{ app }}"{% raw %}; - DROP OWNED BY "{{name}}"; - REVOKE {% endraw %}"{{ app }}"{% raw %} FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM "{{name}}"; - REVOKE USAGE ON SCHEMA public FROM "{{name}}"; - DROP USER "{{name}}"; - {% endraw %} - postgresql_{{ env }}_{{ app }}: - backend: postgres-{{ env }}-{{ app }} - name: {{ app }} - options: - db_name: {{ app }} - default_ttl: {{ SIX_MONTHS }} - max_ttl: {{ SIX_MONTHS }} - creation_statements: >- - {% raw %}CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'{% endraw %} IN ROLE "{{ app }}" INHERIT; - GRANT {% raw %}"{{name}}"{% endraw %} TO odldevops WITH ADMIN OPTION; - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO {% raw %}"{{name}}"{% endraw %} WITH GRANT OPTION; - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO {% raw %}"{{name}}"{% endraw %} WITH GRANT OPTION; - ALTER DEFAULT PRIVILEGES FOR USER {% raw %}"{{name}}"{% endraw %} IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO "{{ app }}" WITH GRANT OPTION; - ALTER DEFAULT PRIVILEGES FOR USER {% raw %}"{{name}}"{% endraw %} IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO "{{ app }}" WITH GRANT OPTION; - {% raw %} - revocation_statements: >- - GRANT "{{name}}" TO odldevops WITH ADMIN OPTION; - REASSIGN OWNED BY "{{name}}" TO {% endraw %}"{{ app }}"{% raw %}; - DROP OWNED BY "{{name}}"; - REVOKE {% endraw %}"{{ app }}"{% raw %} FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM "{{name}}"; - REVOKE USAGE ON SCHEMA public FROM "{{name}}"; - DROP USER "{{name}}"; - {% endraw %} - postgresql_{{ env }}_{{ app }}_readonly: - backend: postgres-{{ env }}-{{ app }} - name: readonly - options: - db_name: {{ app }} - default_ttl: {{ SIX_MONTHS }} - max_ttl: {{ SIX_MONTHS }} - creation_statements: >- - {% raw %}CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'{% endraw %}; - GRANT {% raw %}"{{name}}"{% endraw %} TO odldevops; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO {% raw %}"{{name}}";{% endraw %} - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO {% raw %}"{{name}}";{% endraw %} - ALTER DEFAULT PRIVILEGES FOR USER {% raw %}"{{name}}"{% endraw %} IN SCHEMA public GRANT SELECT ON TABLES TO "{{ app }}" WITH GRANT OPTION; - ALTER DEFAULT PRIVILEGES FOR USER {% raw %}"{{name}}"{% endraw %} IN SCHEMA public GRANT SELECT ON SEQUENCES TO "{{ app }}" WITH GRANT OPTION; - {% raw %} - revocation_statements: >- - GRANT "{{name}}" TO odldevops WITH ADMIN OPTION; - REASSIGN OWNED BY "{{name}}" TO {% endraw %}"{{ app }}"{% raw %}; - DROP OWNED BY "{{name}}"; - REVOKE {% endraw %}"{{ app }}"{% raw %} FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM "{{name}}"; - REVOKE USAGE ON SCHEMA public FROM "{{name}}"; - DROP USER "{{name}}"; - {% endraw %} - {% endfor %}{# End of app loop #} - {% for app in ['starcellbio'] %} - mariadb-{{ env }}-{{ app }}-admin: - backend: mariadb-{{ env }}-{{ app }} - name: admin - options: - db_name: {{ app }} - default_ttl: {{ SIX_MONTHS }} - max_ttl: {{ SIX_MONTHS }} - creation_statements: {% raw %}"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT ALL ON `%`.* TO '{{name}}'@'%';"{% endraw %} - revocation_statements: {% raw %}"DROP USER '{{name}}';"{% endraw %} - mariadb-{{ env }}-{{ app }}-readonly: - backend: mariadb-{{ env }}-{{ app }} - name: readonly - options: - db_name: {{ app }} - default_ttl: {{ SIX_MONTHS }} - max_ttl: {{ SIX_MONTHS }} - creation_statements: {% raw %}"CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';GRANT SELECT, SHOW VIEW ON `%`.* TO '{{name}}'@'%';"{% endraw %} - revocation_statements: {% raw %}"DROP USER '{{name}}';"{% endraw %} - mariadb-{{ env }}-{{ app }}: - backend: mariadb-{{ env }}-{{ app }} - name: {{ app }} - options: - db_name: {{ app }} - default_ttl: {{ SIX_MONTHS }} - max_ttl: {{ SIX_MONTHS }} - creation_statements: "CREATE USER {% raw %}'{{name}}'@'%'{% endraw %} IDENTIFIED BY {% raw %}'{{password}}'{% endraw %};GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES ON {{ app }}.* TO {% raw %}'{{name}}'{% endraw %}@'%';" - revocation_statements: {% raw %}"DROP USER '{{name}}';"{% endraw %} - {% endfor %} - {% endfor %}{# End of env loop #} diff --git a/pillar/vault/roles/aws.sls b/pillar/vault/roles/aws.sls deleted file mode 100644 index 551c025a2..000000000 --- a/pillar/vault/roles/aws.sls +++ /dev/null @@ -1,158 +0,0 @@ -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} - -vault: - roles: - {% for env in ['ci', 'rc', 'production'] %} - {% load_json as ovs_policy %} - { - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:ListBucket", - "s3:HeadObject", - "s3:GetObject" - ], - "Resource": [ - "arn:aws:s3:::ttv_videos", - "arn:aws:s3:::ttv_videos/*", - "arn:aws:s3:::ttv_static", - "arn:aws:s3:::ttv_static/*" - ] - }, - { - "Resource": "*", - "Action": [ - "elastictranscoder:Read*", - "elastictranscoder:List*", - "elastictranscoder:*Job", - "elastictranscoder:*Preset", - "iam:List*", - "sns:List*", - "sns:Publish" - ], - "Effect": "Allow" - }, - { - "Resource": [ - "arn:aws:s3:::odl-video-service*", - "arn:aws:s3:::odl-video-service*/*" - ], - "Action": [ - "s3:HeadObject", - "s3:GetObject", - "s3:ListAllMyBuckets", - "s3:ListBucket", - "s3:ListObjects", - "s3:PutObject", - "s3:DeleteObject" - ], - "Effect": "Allow" - }, - { - "Resource": [ - "arn:aws:s3:::odl-video-service-{{ env }}/", - "arn:aws:s3:::odl-video-service-{{ env }}-transcoded/", - "arn:aws:s3:::odl-video-service-{{ env }}-thumbnails/", - "arn:aws:s3:::odl-video-service-{{ env }}-subtitles/", - "arn:aws:s3:::odl-video-service-{{ env }}/*", - "arn:aws:s3:::odl-video-service-{{ env }}-transcoded/*", - "arn:aws:s3:::odl-video-service-{{ env }}-thumbnails/*", - "arn:aws:s3:::odl-video-service-{{ env }}-subtitles/*" - ], - "Action": [ - "s3:DeleteObject", - "s3:DeleteObjectVersion", - "s3:HeadObject", - "s3:GetAccelerateConfiguration", - "s3:GetBucketAcl", - "s3:GetBucketCORS", - "s3:GetBucketLocation", - "s3:GetBucketLogging", - "s3:GetBucketNotification", - "s3:GetBucketPolicy", - "s3:GetBucketTagging", - "s3:GetBucketVersioning", - "s3:GetBucketWebsite", - "s3:GetLifecycleConfiguration", - "s3:GetObject", - "s3:GetObjectAcl", - "s3:GetObjectTagging", - "s3:GetObjectTorrent", - "s3:GetObjectVersion", - "s3:GetObjectVersionAcl", - "s3:GetObjectVersionTagging", - "s3:GetObjectVersionTorrent", - "s3:GetReplicationConfiguration", - "s3:ListAllMyBuckets", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListBucketVersions", - "s3:ListMultipartUploadParts", - "s3:PutBucketWebsite", - "s3:PutObject", - "s3:PutObjectTagging", - "s3:ReplicateDelete", - "s3:ReplicateObject", - "s3:RestoreObject" - ], - "Effect": "Allow", - "Sid": "Stmt1496679856000" - } - ], - "Version": "2012-10-17" - } - {% endload %} - odl_video_iam_role_for_{{ env }}: - backend: aws-mitx - name: odl-video-service-{{ env }} - options: - policy_document: '{{ ovs_policy|json }}' - credential_type: iam_user - - {% load_json as mit_open_policy %} - { - "Statement": [ - { - "Resource": [ - "arn:aws:s3:::odl-discussions-{{ env }}", - "arn:aws:s3:::odl-discussions-{{ env }}/*", - "arn:aws:s3:::open-learning-course-data-{{ env }}", - "arn:aws:s3:::open-learning-course-data-{{ env }}/*" - ], - "Action": [ - "s3:HeadObject", - "s3:Get*", - "s3:List*", - "s3:Put*", - "S3:DeleteObject" - ], - "Effect": "Allow" - }, - { - "Resource": [ - "arn:aws:s3:::mitx-etl-xpro-production-mitxpro-production", - "arn:aws:s3:::mitx-etl-xpro-production-mitxpro-production/*", - "arn:aws:s3:::ol-olx-course-exports", - "arn:aws:s3:::ol-olx-course-exports/*", - "arn:aws:s3:::ocw-content-storage", - "arn:aws:s3:::ocw-content-storage/*" - ], - "Action": [ - "s3:HeadObject", - "s3:Get*", - "s3:List*" - ], - "Effect": "Allow" - } - ], - "Version": "2012-10-17" - } - {% endload %} - mit_open_iam_role_for_{{ env }}: - backend: aws-mitx - name: mit-open-{{ env }} - options: - policy_document: '{{ mit_open_policy|json }}' - credential_type: iam_user - {% endfor %} diff --git a/pillar/vault/roles/bootcamps.sls b/pillar/vault/roles/bootcamps.sls deleted file mode 100644 index b94420147..000000000 --- a/pillar/vault/roles/bootcamps.sls +++ /dev/null @@ -1,29 +0,0 @@ -vault: - roles: - bootcamps-app: - backend: postgresql-bootcamps - name: app - options: - {% raw %} - sql: >- - CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}' IN ROLE "bootcamp-ecommerce" INHERIT; - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "bootcamp-ecommerce" WITH GRANT OPTION; - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO "bootcamp-ecommerce" WITH GRANT OPTION; - SET ROLE "bootcamp-ecommerce"; - ALTER DEFAULT PRIVILEGES FOR ROLE "bootcamp-ecommerce" IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO "bootcamp-ecommerce" WITH GRANT OPTION; - ALTER DEFAULT PRIVILEGES FOR ROLE "bootcamp-ecommerce" IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO "bootcamp-ecommerce" WITH GRANT OPTION; - RESET ROLE; - ALTER ROLE "{{name}}" SET ROLE "bootcamp-ecommerce"; - {% endraw %} - bootcamps-readonly: - backend: postgresql-bootcamps - name: readonly - options: - {% raw %} - sql: >- - CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{{name}}"; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO "{{name}}"; - {% endraw %} diff --git a/pillar/vault/roles/macros.jinja b/pillar/vault/roles/macros.jinja deleted file mode 100644 index a1bf7725d..000000000 --- a/pillar/vault/roles/macros.jinja +++ /dev/null @@ -1,40 +0,0 @@ -{% macro pg_app_user(approle) %} - sql: >- - {% raw %}CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';{% endraw %} - GRANT {% raw %}"{{name}}"{% endraw %} TO "{{ approle }}" WITH ADMIN OPTION; - GRANT "{{ approle }}" TO {% raw %}"{{name}}";{% endraw %} - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO {% raw %}"{{name}}";{% endraw %} - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO {% raw %}"{{name}}";{% endraw %} - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO {% raw %}"{{name}}";{% endraw %} - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO {% raw %}"{{name}}";{% endraw %} - revocation_sql: >- - REASSIGN OWNED BY {% raw %}"{{name}}"{% endraw %} TO "{{ approle }}"; - DROP OWNED BY {% raw %}"{{name}}"{% endraw %}; - REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM {% raw %}"{{name}}"{% endraw %}; - REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM {% raw %}"{{name}}"{% endraw %}; - ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM {% raw %}"{{name}}"{% endraw %}; - ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON SEQUENCES FROM {% raw %}"{{name}}"{% endraw %}; - REVOKE USAGE ON SCHEMA public FROM {% raw %}"{{name}}"{% endraw %}; - DROP USER {% raw %}"{{name}}"{% endraw %}; -{% endmacro %} - -{% macro pg_readonly(approle) %} - {% raw %} - sql: >- - CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; - GRANT "{{name}}" TO odldevops WITH ADMIN OPTION; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{{name}}"; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO "{{name}}"; - revocation_sql: >- - REASSIGN OWNED BY "{{name}}" TO "bootcamp-ecommerce"; - DROP OWNED BY "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM "{{name}}"; - REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON TABLES FROM "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE ALL ON SEQUENCES FROM "{{name}}"; - REVOKE USAGE ON SCHEMA public FROM "{{name}}"; - DROP USER "{{name}}"; - {% endraw %} -{% endmacro %} diff --git a/pillar/vault/roles/micromasters.sls b/pillar/vault/roles/micromasters.sls deleted file mode 100644 index bcb3a3bee..000000000 --- a/pillar/vault/roles/micromasters.sls +++ /dev/null @@ -1,26 +0,0 @@ -vault: - roles: - micromasters-app: - backend: postgresql-micromasters - name: app - options: - {% raw %} - sql: >- - CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "{{name}}"; - GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO "{{name}}"; - {% endraw %} - micromasters-readonly: - backend: postgresql-micromasters - name: readonly - options: - {% raw %} - sql: >- - CREATE USER "{{name}}" WITH PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{{name}}"; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO "{{name}}"; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO "{{name}}"; - {% endraw %} diff --git a/pillar/vector/odlvideo.sls b/pillar/vector/odlvideo.sls deleted file mode 100644 index e503145e1..000000000 --- a/pillar/vector/odlvideo.sls +++ /dev/null @@ -1,10 +0,0 @@ -vector: - configurations: - - host_metrics - - auth_logs - - nginx_logs - - odlvideo_logs - - config_elements: - application_name: 'odlvideo' - service_name: 'odlvideo' diff --git a/salt/apps/ocw/nextgen_build_install.sls b/salt/apps/ocw/nextgen_build_install.sls deleted file mode 100644 index 3c763947b..000000000 --- a/salt/apps/ocw/nextgen_build_install.sls +++ /dev/null @@ -1,122 +0,0 @@ -{% set ocw_next = salt.pillar.get('ocw-next') %} - -manage_yarn_pkg_repo: - pkgrepo.managed: - - name: deb https://dl.yarnpkg.com/debian/ stable main - - key_url: https://dl.yarnpkg.com/debian/pubkey.gpg - -ensure_os_package_prerequisites: - pkg.installed: - - refresh: True - - pkgs: - - awscli - - git - - build-essential - - gcc - - g++ - - make - - yarn - - jq - - golang - -ensure_state_of_hugo_binary: - pkg.installed: - - refresh: True - - sources: - - hugo: https://github.com/gohugoio/hugo/releases/download/v0.80.0/hugo_0.80.0_Linux-64bit.deb - -ensure_state_of_caddy_home: - # the formula doesn't create a caddy home dir, but we actually want it - # because we need a place for dotfiles, etc., that are created by - # npm. - file.directory: - - name: /home/caddy - - user: caddy - - group: caddy - -ensure_state_of_opt_ocw: - file.directory: - - name: /opt/ocw - - user: caddy - - group: caddy - - dir_mode: '0755' - -ensure_state_of_log_directory: - file.directory: - - name: /opt/ocw/logs - - user: caddy - - group: caddy - - dir_mode: '0755' - -git_pull_ocw_www: - git.latest: - - name: https://github.com/mitodl/ocw-www.git - - target: /opt/ocw/ocw-www - - rev: {{ ocw_next.ocw_www_git_ref }} - - force_checkout: True - - force_clone: True - - force_reset: True - - force_fetch: True - - update_head: True - - user: caddy - - require: - - pkg: ensure_os_package_prerequisites - -git_pull_ocw_hugo_themes: - git.latest: - - name: https://github.com/mitodl/ocw-hugo-themes.git - - target: /opt/ocw/ocw-hugo-themes - - rev: {{ ocw_next.ocw_hugo_themes_git_ref }} - - force_checkout: True - - force_clone: True - - force_reset: True - - force_fetch: True - - update_head: True - - user: caddy - - require: - - pkg: ensure_os_package_prerequisites - -manage_ocw_www_env_file: - file.managed: - - name: /opt/ocw/ocw-www/.env - - user: caddy - - group: caddy - - mode: 0640 - - contents: | - SEARCH_API_URL={{ ocw_next.search_api_url }} - - require: - - git: git_pull_ocw_www - -manage_ocw_hugo_themes_env_file: - file.managed: - - name: /opt/ocw/ocw-hugo-themes/.env - - user: caddy - - group: caddy - - mode: 0640 - - contents: | - SEARCH_API_URL={{ ocw_next.search_api_url }} - - require: - - git: git_pull_ocw_hugo_themes - -install_caddy_webhook_script: - file.managed: - - name: /opt/ocw/webhook-publish.sh - - user: caddy - - group: caddy - - mode: 0700 - - source: salt://apps/ocw/templates/webhook-publish.sh.jinja - - template: jinja - - context: - website_bucket: {{ ocw_next.website_bucket }} - ocw_to_hugo_bucket: {{ ocw_next.ocw_to_hugo_bucket }} - source_data_bucket: {{ ocw_next.source_data_bucket }} - fastly_api_token: {{ ocw_next.fastly_api_token }} - fastly_service_id: {{ ocw_next.fastly_service_id }} - ocw_www_git_ref: {{ ocw_next.ocw_www_git_ref }} - ocw_hugo_themes_git_ref: {{ ocw_next.ocw_hugo_themes_git_ref }} - ocw_hugo_projects_git_ref: {{ ocw_next.ocw_hugo_projects_git_ref }} - course_base_url: {{ ocw_next.course_base_url }} - ocw_studio_base_url: {{ ocw_next.ocw_studio_base_url }} - gtm_account_id: {{ ocw_next.gtm_account_id }} - ocw_import_starter_slug: {{ ocw_next.ocw_import_starter_slug }} - static_api_base_url: {{ ocw_next.static_api_base_url }} diff --git a/salt/apps/ocw/templates/webhook-publish.sh.jinja b/salt/apps/ocw/templates/webhook-publish.sh.jinja deleted file mode 100644 index 6bb2a295b..000000000 --- a/salt/apps/ocw/templates/webhook-publish.sh.jinja +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env bash - -# Usage -# -# To run a normal publish, running only if code has changed: -# -# cd /opt/ocw -# ./webhook-publish.sh -# -# OR, to publish everything, no matter if any code has changed: -# -# cd /opt/ocw -# ./webhook-publish.sh full - -LOG_DIR=/opt/ocw/logs -LOG_FILE=$LOG_DIR/webhook-publish.log -SOURCE_DATA_BUCKET={{ source_data_bucket }} -SOURCE_DATA_DIR=/opt/ocw/open-learning-course-data -OCW_HUGO_PROJECTS_PATH=/opt/ocw/ocw-hugo-projects -OCW_HUGO_THEMES_PATH=/opt/ocw/ocw-hugo-themes -WWW_HUGO_CONFIG_PATH=$OCW_HUGO_PROJECTS_PATH/ocw-www/config.yaml -COURSE_HUGO_CONFIG_PATH=$OCW_HUGO_PROJECTS_PATH/ocw-course/config.yaml -WWW_CONTENT_PATH=/opt/ocw/ocw-www -SITE_OUTPUT_DIR=$WWW_CONTENT_PATH/public/ # Should end in '/' -COURSE_CONTENT_PATH=/opt/ocw/ocw-to-hugo/private/output -WEBSITE_BUCKET={{ website_bucket }} -OCW_TO_HUGO_BUCKET={{ ocw_to_hugo_bucket }} -FASTLY_API_TOKEN={{ fastly_api_token }} -FASTLY_SERVICE_ID={{ fastly_service_id }} -OCW_WWW_GIT_REF={{ ocw_www_git_ref }} -OCW_HUGO_THEMES_GIT_REF={{ ocw_hugo_themes_git_ref }} -OCW_HUGO_PROJECTS_GIT_REF={{ ocw_hugo_projects_git_ref }} -COURSE_BASE_URL={{ course_base_url }} -OCW_STUDIO_BASE_URL={{ ocw_studio_base_url }} -OCW_IMPORT_STARTER_SLUG={{ ocw_import_starter_slug }} -STATIC_API_BASE_URL={{ static_api_base_url }} -GTM_ACCOUNT_ID={{ gtm_account_id }} -OCW_HASH_DIR=$WWW_CONTENT_PATH/public/static -OCW_WWW_HASH_PATH=$WWW_CONTENT_PATH/public/static/ocw-www-hash.txt -OCW_HUGO_THEMES_HASH_PATH=$WWW_CONTENT_PATH/public/static/ocw-hugo-themes-hash.txt -OCW_HUGO_PROJECTS_HASH_PATH=$WWW_CONTENT_PATH/public/static/ocw-hugo-projects-hash.txt -# lock_dir ensures that only one run of this script happens at once. -lock_dir=/tmp/webhook-publish-lock -# If retry_file is present, the script will run itself again to catch changes -# that came in during the period of the current run. -retry_file=/tmp/webhook-publish-retry - -# Export those shell variables that must be environment variables for the -# processes we execute -export GTM_ACCOUNT_ID -export WWW_CONTENT_PATH=$WWW_CONTENT_PATH -export COURSE_CONTENT_PATH=$COURSE_CONTENT_PATH -export COURSE_HUGO_CONFIG_PATH=$COURSE_HUGO_CONFIG_PATH -export COURSE_BASE_URL=$COURSE_BASE_URL -export OCW_STUDIO_BASE_URL=$OCW_STUDIO_BASE_URL -export OCW_IMPORT_STARTER_SLUG=$OCW_IMPORT_STARTER_SLUG -export STATIC_API_BASE_URL=$STATIC_API_BASE_URL - -# Optional script argument -script_option=$1 - -log_message() { - echo `date +'%Y-%m-%d %H:%M:%S.%N'` $1 | tee -a ${LOG_FILE} -} - -error_and_exit() { - log_message $1 - rm -rf $lock_dir - exit 1 -} - -# Do a git fetch and git reset to accomplish a `git pull` without failing if -# there was a change to the git history. -# Return 0 if the working copy was changed, 1 if it was not. -# -git_fetch_and_reset() { - repo_name=$1 - git_ref=$2 - orig_commit=`git rev-parse HEAD` - log_message "Pulling $repo_name" - git fetch && git reset --hard origin/$git_ref \ - || error_and_exit "Can not pull $repo_name" - git clean -f || error_and_exit "Unable to run git clean on directory for $repo_name" - new_commit=`git rev-parse HEAD` - log_message "$orig_commit -> $new_commit" - if [ "$orig_commit" != "$new_commit" ]; then - log_message "... Pulled new commit" - return 0 - else - log_message "... No new commit" - return 1 - fi -} - -clear_directory() { - dir=$1 - log_message "Clearing $dir" - rm -rf $dir - if [ $? -ne 0 ]; then - error_and_exit "Could not clear $dir" - fi -} - -# Manage locking directory and retry file - -mkdir $lock_dir -if [ $? -ne 0 ]; then - echo "Can not acquire lock. Another run in-progress?" >&2 - touch $retry_file - exit 1 -fi - -if [ -e $retry_file ]; then - log_message "This is a re-run." - rm $retry_file -fi - - -# Pull source data - -log_message "Pulling source data" -aws s3 sync s3://$SOURCE_DATA_BUCKET/ $SOURCE_DATA_DIR/ --delete \ - --only-show-errors -if [ $? -ne 0 ]; then - error_and_exit "Failed to pull source data" -fi - - -# Pull ocw-www -cd $WWW_CONTENT_PATH || error_and_exit "Can not cd to ocw-www" -git_fetch_and_reset ocw-www $OCW_WWW_GIT_REF -ocw_www_changed=$? - -# Pull ocw-hugo-themes -cd $OCW_HUGO_THEMES_PATH || error_and_exit "Can not cd to ocw-hugo-themes" -git_fetch_and_reset ocw-hugo-themes $OCW_HUGO_THEMES_GIT_REF -ocw_hugo_themes_changed=$? - -# Pull ocw-hugo-projects -cd $OCW_HUGO_PROJECTS_PATH || error_and_exit "Can not cd to ocw-hugo-projects" -git_fetch_and_reset ocw-hugo-projects $OCW_HUGO_PROJECTS_GIT_REF -ocw_hugo_projects_changed=$? - -# Continue if there were changes, or if we want a 'full' run regardless of -# changes. -if [ "$script_option" != "full" ]; then - if [ $ocw_www_changed -eq 0 ] || [ $ocw_hugo_themes_changed -eq 0 ]; then - - log_message "Pulled new commit; continuing with publish." - - else - - log_message "No new commits; stopping." - rmdir $lock_dir - exit 0 - fi -fi - -# Install packages for ocw-hugo-themes (this should include ocw-to-hugo) -cd $OCW_HUGO_THEMES_PATH -log_message "Doing yarn install of dependencies for ocw-hugo-themes" -yarn install --pure-lockfile >> $LOG_DIR/ocw-hugo-themes-install.log 2>&1 \ - || error_and_exit "Can't install dependencies for ocw-hugo-themes" - -# Generate ocw-hugo-themes git hash -npm run build:githash || error_and_exit "Unable to create hash file" - -# Build ocw-www using ocw-hugo-themes -npm run build -- $WWW_CONTENT_PATH $WWW_HUGO_CONFIG_PATH || error_and_exit "Unable to build ocw-www site" - -# Make sure output directory is clear and move built ocw-www there -mkdir -p $SITE_OUTPUT_DIR -find $SITE_OUTPUT_DIR -mindepth 1 -delete -mv $WWW_CONTENT_PATH/dist/* $SITE_OUTPUT_DIR - -# Run ocw-to-hugo -cd ./node_modules/@mitodl/ocw-to-hugo - -clear_directory ./node_modules - -log_message "Doing yarn install of ocw-to-hugo" -yarn install --pure-lockfile >> $LOG_DIR/ocw-to-hugo-install.log 2>&1 \ - || error_and_exit "Can not install ocw-to-hugo" - -log_message "Running ocw-to-hugo" -node . -i $SOURCE_DATA_DIR -o $COURSE_CONTENT_PATH \ - --strips3 --staticPrefix /coursemedia --rm >> $LOG_DIR/ocw-to-hugo.log 2>&1 -if [ $? -ne 0 ]; then - error_and_exit "Failed to run ocw-to-hugo. See $LOG_DIR/ocw-to-hugo.log" -fi - - -# Run course builds with ocw-hugo-themes -echo "Running hugo on courses in $COURSE_CONTENT_PATH..." -cd /opt/ocw/ocw-hugo-themes -./package_scripts/build_all_courses.sh >> $LOG_DIR/course-builds.log 2>&1 -if [ $? -ne 0 ]; then - error_and_exit "Failed to run course builds. See $LOG_DIR/course-builds.log" -fi - -# Write commit hash files - -log_message "Writing commit hash files" - -cd $WWW_CONTENT_PATH \ -&& git rev-parse HEAD > $OCW_WWW_HASH_PATH \ -&& cd $OCW_HUGO_THEMES_PATH \ -&& git rev-parse HEAD > $OCW_HUGO_THEMES_HASH_PATH \ -&& cd $OCW_HUGO_PROJECTS_PATH \ -&& git rev-parse HEAD > $OCW_HUGO_PROJECTS_HASH_PATH - -if [ $? -ne 0 ]; then - error_and_exit "Failed to write commit hash files" -fi - - -# Sync HTML to S3 bucket - -log_message "Syncing to S3 bucket ($WEBSITE_BUCKET)" -# a little double-check to make sure the source directory ends in a slash, to -# prevent the site from getting copied into a subdirectory ... -echo "$SITE_OUTPUT_DIR" | grep -q '/$' -if [ $? -ne 0 ]; then - log_message "WARNING: appending '/' to $SITE_OUTPUT_DIR" - SITE_OUTPUT_DIR=$SITE_OUTPUT_DIR/ -fi -aws s3 sync $SITE_OUTPUT_DIR s3://$WEBSITE_BUCKET/ \ - --delete --only-show-errors >> $LOG_DIR/website-sync.log 2>&1 -if [ $? -ne 0 ]; then - error_and_exit "Failed to sync to S3 bucket. See $LOG_DIR/website-sync.log" -fi - - -# Clear CDN cache - -log_message "Clearing Fastly cache" -curl -f -X POST -H "Fastly-Key: $FASTLY_API_TOKEN" \ - https://api.fastly.com/service/$FASTLY_SERVICE_ID/purge_all -if [ $? -ne 0 ]; then - log_message "WARNING: Failed to clear Fastly cache!" -fi - - -log_message "Done" - -# Clean up - -if [ -e $retry_file ]; then - log_message "$retry_file exists. Removing $lock_dir and re-running ..." - rmdir $lock_dir - exec /opt/ocw/webhook-publish.sh -else - rmdir $lock_dir -fi - -exit 0 diff --git a/salt/apps/odlvideo/configure.sls b/salt/apps/odlvideo/configure.sls deleted file mode 100644 index 8a507d7d0..000000000 --- a/salt/apps/odlvideo/configure.sls +++ /dev/null @@ -1,32 +0,0 @@ -include: - - uwsgi.service - - apps.odlvideo.deploy_signal - -create_env_file_for_odlvideo: - file.managed: - - name: /opt/{{ salt.pillar.get('django:app_name') }}/.env - - contents: | - {%- for var, val in salt.pillar.get('django:environment').items() %} - {{ var }}={{ val }} - {%- endfor %} - - onchanges_in: - - service: uwsgi_service_running - - file: signal_odlvideo_deploy_complete - -ensure_perms_of_odlvideo_log_dir: - file.directory: - - name: /var/log/odl-video/ - - user: deploy - - group: deploy - - dir_mode: 0755 - - onchanges_in: - - service: uwsgi_service_running - -ensure_perms_of_odlvideo_app_log: - file.managed: - - name: /var/log/odl-video/django.log - - user: deploy - - group: deploy - - mode: 0644 - - onchanges_in: - - service: uwsgi_service_running diff --git a/salt/apps/odlvideo/deploy_signal.sls b/salt/apps/odlvideo/deploy_signal.sls deleted file mode 100644 index d371d56f2..000000000 --- a/salt/apps/odlvideo/deploy_signal.sls +++ /dev/null @@ -1,6 +0,0 @@ -{% set app_dir = '/opt/{0}'.format(salt.pillar.get('django:app_name')) %} - -signal_odlvideo_deploy_complete: - file.touch: - - name: {{ app_dir }}/deploy_complete.txt - - order: last diff --git a/salt/apps/odlvideo/install.sls b/salt/apps/odlvideo/install.sls deleted file mode 100644 index f785229fe..000000000 --- a/salt/apps/odlvideo/install.sls +++ /dev/null @@ -1,11 +0,0 @@ -ensure_yarn_is_installed_for_odlvideo: - npm.installed: - - name: 'yarn@1.22.4' - - user: root - -install_node_dependencies: - cmd.run: - - name: yarn install - - cwd: /opt/{{ salt.pillar.get('django:app_name') }} - - require: - - deploy_application_source_to_destination diff --git a/salt/apps/odlvideo/post_deploy.sls b/salt/apps/odlvideo/post_deploy.sls deleted file mode 100644 index eb5df9561..000000000 --- a/salt/apps/odlvideo/post_deploy.sls +++ /dev/null @@ -1,15 +0,0 @@ -{% set app_dir = '/opt/{0}'.format(salt.pillar.get('django:app_name')) %} - -build_static_assets_for_odlvideo: - cmd.script: - - name: {{ app_dir }}/webpack_if_prod.sh - - cwd: {{ app_dir }} - - env: - - NODE_ENV: production - - user: deploy - -generate_deploy_hash_for_odlvideo: - cmd.run: - - name: 'git log --pretty=format:%H -n 1 > static/hash.txt' - - cwd: {{ app_dir }} - - user: deploy diff --git a/salt/apps/starcellbio/config.sls b/salt/apps/starcellbio/config.sls deleted file mode 100644 index 1d235faf2..000000000 --- a/salt/apps/starcellbio/config.sls +++ /dev/null @@ -1,13 +0,0 @@ -{% set app_dir = '/opt/{0}'.format(salt.pillar.get('django:app_name')) %} - -include: - - uwsgi.service - -write_app_config_overrides: - file.managed: - - name: {{ app_dir }}/StarCellBio/settings.yml - - contents: | - {{ salt.pillar.get('starcellbio:config', {})|yaml(False)|indent(8) }} - - user: deploy - - onchanges_in: - - service: uwsgi_service_running diff --git a/salt/apps/starcellbio/install.sls b/salt/apps/starcellbio/install.sls deleted file mode 100644 index 1d12b4dfd..000000000 --- a/salt/apps/starcellbio/install.sls +++ /dev/null @@ -1,25 +0,0 @@ -{% set app_dir = '/opt/{0}'.format(salt.pillar.get('django:app_name')) %} - -install_node_dependencies: - npm.bootstrap: - - name: {{ app_dir }} - - require: - - git: deploy_application_source_to_destination - - pkg: django_system_dependencies - -download_jqdialog_dependency: - file.managed: - - name: {{ app_dir }}/html_app/js/jqdialog.js - - source: https://raw.githubusercontent.com/knadh/jqdialog/f8dc7e4dca84ab132448723d3be35124d7de4fbc/jqdialog.js - - source_hash: 2f12e880659e0b0092e3a5a7cf7f8bdbeb707b8b649a0a9cd5c263c7362c0b53 - - user: deploy - - require: - - git: deploy_application_source_to_destination - -install_soyutils_dependency: - file.copy: - - name: {{ app_dir }}/html_app/js/soyutils.js - - source: {{ app_dir }}/node_modules/closure-templates/soyutils.js - - user: deploy - - require: - - npm: install_node_dependencies diff --git a/salt/apps/starcellbio/post_deploy.sls b/salt/apps/starcellbio/post_deploy.sls deleted file mode 100644 index d7ce1e481..000000000 --- a/salt/apps/starcellbio/post_deploy.sls +++ /dev/null @@ -1,26 +0,0 @@ -{% set app_dir = '/opt/{0}'.format(salt.pillar.get('django:app_name')) %} -{% set django = salt.pillar.get('django') %} - -populate_database_with_seed_data: - module.run: - - name: django.loaddata - - settings_module: {{ django.settings_module }} - - pythonpath: {{ app_dir }} - - fixtures: auth,backend,courses,assignments,studentassignments - - bin_env: {{ django.django_admin_path }} - - runas: deploy - - env: {{ django.get('environment', {})|tojson }} - {% if django.automatic_migrations %} - - require: - - module: migrate_database - {% endif %} - -compile_static_files: - cmd.run: - - name: /usr/local/pyenv/shims/python html_app/build.py - - cwd: {{ app_dir }} - - shell: /bin/bash - - user: deploy - - prepend_path: {{ app_dir }}/node_modules/.bin - - env: - PROJECT_HOME: {{ app_dir }}/html_app diff --git a/salt/edx/django_user.sls b/salt/edx/django_user.sls deleted file mode 100644 index a6e3df583..000000000 --- a/salt/edx/django_user.sls +++ /dev/null @@ -1,22 +0,0 @@ -{% set django_superuser_account = salt.pillar.get('devstack:edx:django:django_superuser_account', 'devstack') %} -{% set django_superuser_password = salt.pillar.get('devstack:edx:django:django_superuser_password', 'changeme') %} - -create_django_superuser_account: - cmd.run: - - name: /edx/bin/python.edxapp /edx/bin/manage.edxapp lms manage_user {{ django_superuser_account }} {{ django_superuser_account }}@example.com --staff --superuser --settings=devstack - - cwd: /edx/app/edxapp/edx-platform/ - - runas: edxapp - -create_django_staff_account: - cmd.run: - - name: /edx/bin/python.edxapp /edx/bin/manage.edxapp lms create_user -u staff -e staff@example.com -p {{ django_superuser_password }} --staff --settings=devstack - - cwd: /edx/app/edxapp/edx-platform/ - - runas: edxapp - -{% for account in ["audit", "honor", "verified"] %} -create_django_{{ account }}_account: - cmd.run: - - name: /edx/bin/python.edxapp /edx/bin/manage.edxapp lms create_user -u {{ account }} -e {{ account }}@example.com -p {{ django_superuser_password }} --settings=devstack - - cwd: /edx/app/edxapp/edx-platform/ - - runas: edxapp -{% endfor %} diff --git a/salt/edx/edxapp_global_pre_commit.sls b/salt/edx/edxapp_global_pre_commit.sls deleted file mode 100644 index 991a0c603..000000000 --- a/salt/edx/edxapp_global_pre_commit.sls +++ /dev/null @@ -1,8 +0,0 @@ -{# edX has a bug that results in course exports removing the run - as part of the course data, so we need to run a pre-commit git - hook during the export to add that data back to the course content #} -install_pre_commit_template_for_course_export: - file.managed: - - name: /usr/share/git-core/templates/hooks/pre-commit - - source: salt://edx/files/edx_export_pre_commit.sh - - mode: 0755 diff --git a/salt/edx/etc_hosts.sls b/salt/edx/etc_hosts.sls deleted file mode 100644 index 0a4ac2522..000000000 --- a/salt/edx/etc_hosts.sls +++ /dev/null @@ -1,9 +0,0 @@ -#!jinja|yaml - -{% set lms_site_name = salt.pillar.get('edx:ansible_vars:EDXAPP_LMS_SITE_NAME') %} - -add_etc_hosts_entry: - host.present: - - ip: 127.0.0.1 - - names: - - {{ lms_site_name }} diff --git a/salt/edx/files/edx_export_pre_commit.sh b/salt/edx/files/edx_export_pre_commit.sh deleted file mode 100644 index 0e79aba41..000000000 --- a/salt/edx/files/edx_export_pre_commit.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -# -# Use this pre-commit hook to fix up OLX that is exported from edX without any run information -# -# It assumes that there is a commit already in the repo with a comment of the form: -# -# initial commit of course.xml with term "{run name}" -# -# The script extracts the run from the commit message and then uses it to fix up all the places that need it - -# find the course term -commit_msg=$(git log -1 --grep='initial commit of course.xml with term' --oneline) -run=$(echo $commit_msg| cut -d'"' -f 2) -# abort if we can't find the run -if [ -z $run ] -then - echo "could not find run. proceed with commit" - exit 0 -else - echo "Course run is $run" -fi - -# update the /course.xml -if [ -e course.xml ] -then - echo "replacing url_name in course.xml" - sed -i "s/url_name=\"course\"/url_name=\"$run\"/" course.xml - git add course.xml -fi - -# mv /course/course.xml -if [ -e course/course.xml ] -then - echo "git mv course/course.xml course/$run.xml" - git mv -f course/course.xml course/$run.xml -fi - -# edit polices/course/policy.json -if [ -e policies/course/policy.json ] -then - echo "changing course key in policy.json" - sed -i "s+\"course/course\":+\"course/$run\":+" policies/course/policy.json - git add policies/course/policy.json -fi - -# move /policies/course/ -if [ -d policies/course/ ] -then - echo "git mv policies/course/ policies/$run/" - git rm -r policies/$run - git mv -f policies/course/ policies/$run -fi - -exit 0 diff --git a/salt/edx/files/mitx_devstack.yml b/salt/edx/files/mitx_devstack.yml deleted file mode 100644 index 463bb4d33..000000000 --- a/salt/edx/files/mitx_devstack.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- - -# Stateless app server configuration, designed to be used with external mysql, -# mongo, rabbitmq, and elasticsearch services. - -- name: Bootstrap instance(s) - hosts: all - gather_facts: no - become: True - roles: - - python - -- name: Configure instance(s) - hosts: all - become: True - gather_facts: True - - vars: - migrate_db: 'yes' - openid_workaround: True - EDXAPP_LMS_NGINX_PORT: '80' - ENABLE_ECOMMERCE: False # Disable ecommerce by default - roles: - - # Ensure we have no known security vulnerabilities - - security - - # Server setup - - swapfile - - # Nginx reverse proxy - - role: nginx - nginx_sites: - - cms - - lms - nginx_default_sites: - - lms - - # Main EdX application - # https://github.com/edx/edx-platform - - role: edxapp - celery_worker: True - - edxapp - - # memcached - - role: memcache - when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)" diff --git a/salt/edx/hacks.sls b/salt/edx/hacks.sls deleted file mode 100644 index 5002d6191..000000000 --- a/salt/edx/hacks.sls +++ /dev/null @@ -1,53 +0,0 @@ -{% set purpose = salt.grains.get('purpose', 'xpro-qa') %} - -{% if salt.file.directory_exists('/edx/var/edxapp/staticfiles/studio/templates') %} -ensure_license_selector_template_is_in_expected_location: - file.copy: - - name: /edx/var/edxapp/staticfiles/studio/templates/license-selector.underscore.js - - source: /edx/var/edxapp/staticfiles/studio/templates/license-selector.underscore - - preserve: True -{% endif %} - -{% if salt.file.directory_exists('/edx/var/edxapp/staticfiles/studio/common/templates/components') %} -ensure_system_feedback_template_is_in_expected_location: - file.copy: - - name: /edx/var/edxapp/staticfiles/studio/common/templates/components/system-feedback.underscore.js - - source: /edx/var/edxapp/staticfiles/studio/common/templates/components/system-feedback.underscore - - preserve: True -{% endif %} - -{% if salt.file.directory_exists('/edx/var/edxapp/staticfiles/paragon/static') %} -create_static_assets_subfolder: - file.directory: - - name: /edx/var/edxapp/staticfiles/paragon/static/static - - user: edxapp - - group: edxapp - -copy_select_static_assets_to_static_subfolder: - module.run: - - name: file.copy - - src: /edx/var/edxapp/staticfiles/paragon/static/ - - dst: /edx/var/edxapp/staticfiles/paragon/static/static/ - - recurse: True - - remove_existing: True - - preserve: True -{% endif %} - -{% if 'mitxpro' in salt.grains.get('environment') %} -add_social_auth_https_redirect_to_lms_production_file: - file.append: - - name: /edx/app/edxapp/edx-platform/lms/envs/production.py - - text: SOCIAL_AUTH_REDIRECT_IS_HTTPS = ENV_TOKENS.get('SOCIAL_AUTH_REDIRECT_IS_HTTPS', True) -{% endif %} - -{% if 'residential' in salt.grains.get('purpose') and 'edx-worker' in salt.grains.get('roles') %} -add_cron_task_for_saml_metadata_refresh: - cron.present: - - user: edxapp - - identifier: edx-saml-metadata-refresh - - comment: Periodically pull the SAML metadata so that it doesn't expire and break edX login - - name: . /edx/app/edxapp/edxapp_env && /edx/app/edxapp/venvs/edxapp/bin/python /edx/app/edxapp/edx-platform/manage.py lms saml --pull - - minute: random - - hour: random - - dayweek: random -{% endif %} diff --git a/salt/edx/maintenance_tasks.sls b/salt/edx/maintenance_tasks.sls deleted file mode 100644 index 3286801a9..000000000 --- a/salt/edx/maintenance_tasks.sls +++ /dev/null @@ -1,6 +0,0 @@ -delete_edx_logs_older_than_30_days: - cmd.run: - - name: >- - find /edx/var/log -not -path "/edx/var/log/tracking/*" - -type f \( -name "*.gz" -o -name "lms-stderr.log.*" \) - -mtime +30 -delete diff --git a/salt/edx/migration.sls b/salt/edx/migration.sls deleted file mode 100644 index bad3110da..000000000 --- a/salt/edx/migration.sls +++ /dev/null @@ -1,16 +0,0 @@ -{% set edxapp_bin = '/edx/app/edxapp/venvs/edxapp/bin/python' %} -{% set migrations = ['lms', 'cms'] %} - -{% for migration in migrations %} -run_make_migrations_in_{{ migration }}_for_django_plugins: - cmd.run: - - name: '{{ edxapp_bin }} manage.py {{ migration }} makemigrations' - - cwd: /edx/app/edxapp/edx-platform - - runas: edxapp - -run_edxapp_{{ migration }}_migrations: - cmd.run: - - name: '{{ edxapp_bin }} manage.py {{ migration }} migrate --noinput --fake-initial --settings=production' - - cwd: /edx/app/edxapp/edx-platform - - runas: edxapp -{% endfor %} diff --git a/salt/edx/patch_nginx.sls b/salt/edx/patch_nginx.sls deleted file mode 100644 index 072edf7e8..000000000 --- a/salt/edx/patch_nginx.sls +++ /dev/null @@ -1,21 +0,0 @@ -configure_nginx_status_module_for_edx: - file.managed: - - name: /etc/nginx/sites-enabled/status_monitor - - contents: | - server { - listen 127.0.0.1:80; - location /nginx_status { - stub_status on; - access_log off; - allow 127.0.0.1; - deny all; - } - } - - group: www-data - -reload_edx_nginx_service_after_updates: - service.running: - - name: nginx - - reload: True - - onchanges_any: - - file: configure_nginx_status_module_for_edx diff --git a/salt/edx/prod.sls b/salt/edx/prod.sls deleted file mode 100644 index f9dc1e9de..000000000 --- a/salt/edx/prod.sls +++ /dev/null @@ -1,168 +0,0 @@ -{% set data_path = '/tmp/edx_config' -%} -{% set venv_path = '/tmp/edx_config/venv' -%} -{% set repo_path = '/tmp/edx_config/configuration' -%} -{% set conf_file = '/tmp/edx_config/edx-sandbox.conf' -%} -{% set git_export_path = salt.pillar.get('edxapp:EDXAPP_GIT_REPO_EXPORT_DIR', - '/edx/var/edxapp/export_course_repos') -%} -{% set git_servers = salt.pillar.get('edx:ssh_hosts', - [{'name': 'github.com', - 'fingerprint': '9d:38:5b:83:a9:17:52:92:56:1a:5e:c4:d4:81:8e:0a:ca:51:a2:64:f1:74:20:11:2e:f8:8a:c3:a1:39:49:8f'}, - {'name': 'github.mit.edu', - 'fingerprint': 'aa:d2:e9:66:7e:46:77:d3:7d:d9:39:3f:f4:9f:17:a1:18:c1:87:8f:69:cb:8f:d0:db:10:b7:71:5e:ad:57:68'}]) %} -{% set theme_repo = salt.pillar.get('edx:edxapp:custom_theme:repo', 'https://github.com/mitodl/mitx-theme') -%} -{% set theme_name = salt.pillar.get('edx:edxapp:THEME_NAME', None) -%} -{% set theme_branch = salt.pillar.get('edx:edxapp:custom_theme:branch', 'mitx') -%} -{% set theme_dir = salt.pillar.get('edx:edxapp:EDXAPP_COMPREHENSIVE_THEME_DIR', '/edx/app/edxapp/themes') -%} -{% set os_packages = salt.pillar.get('edx:dependencies:os_packages', - ['git', - 'libmysqlclient-dev', - 'mariadb-client-10.0', - 'landscape-common', - 'libssl-dev', - 'python3-dev', - 'python3-pip', - 'python3-virtualenv', - 'nfs-common', - 'postfix', - 'memcached']) -%} - -include: - - .run_ansible - -install_os_packages: - pkg.installed: - - pkgs: {{ os_packages|tojson }} - - refresh: True - - refresh_modules: True - - require_in: - - virtualenv: create_ansible_virtualenv - - git: clone_edx_configuration - -{% if salt.pillar.get('edx:generate_tls_certificate') %} -generate_self_signed_certificate: - module.run: - - name: tls.create_self_signed_cert - - CN: {{ salt.pillar.get('edx:ansible_env_config:TLS_KEY_NAME') }} - - replace: True - - require_in: - - cmd: run_ansible -{% else %} -{% - set key_path = '{}/{}'.format( - salt.pillar.get('edx:edxapp:TLS_LOCATION'), - salt.pillar.get('edx:edxapp:TLS_KEY_NAME') - ) -%} -{% for ext in ['crt', 'key'] %} -place_tls_{{ ext }}_file: - file.managed: - - name: {{ key_path }}.{{ ext }} - - contents_pillar: {{ 'edx:tls_{}'.format(ext) }} - - user: root - - group: root - - mode: 600 - - makedirs: True - - require_in: - - cmd: run_ansible -{% endfor %} -{% endif %} - - -{# BEGIN states that do not apply to "sandbox" and "devstack" ... #} -{% if not ('sandbox' in salt.grains.get('roles')) %} - -{% set device_name = '{}.efs.us-east-1.amazonaws.com:/'.format(salt.pillar.get('edx:efs_id')) %} -{% set fstab_contents = salt.mount.fstab() %} -{% for fmount, settings in fstab_contents.items() %} -{% if fmount == '/mnt/data' and settings.fstype == 'nfs4' and settings.device != device_name %} -remove_{{ settings.device }}_mount_config_from_fstab: - mount.unmounted: - - name: {{ fmount }} - - device: {{ settings.device }} - - persist: True -{% endif %} -{% endfor %} - -mount_efs_filesystem_for_course_assets: - mount.mounted: - - name: /mnt/data - - device: {{ salt.grains.get('ec2:availability_zone', 'us-east-1b')|trim }}.{{ salt.pillar.get('edx:efs_id')|trim }}.efs.us-east-1.amazonaws.com:/ - - fstype: nfs4 - - mkmnt: True - - persist: True - - mount: True - -create_course_asset_symlink: - file.symlink: - - name: /edx/var/edxapp/course_static - - target: {{ salt.pillar.get('edx:edxapp:GIT_REPO_DIR', '/mnt/data/prod_repos') }} - - makedirs: True - - force: True - - user: edxapp - - group: www-data - -create_edxapp_data_dir_symlink: - file.symlink: - - name: /edx/app/edxapp/data - - target: {{ salt.pillar.get('edx:edxapp:GIT_REPO_DIR', '/mnt/data/prod_repos') }} - - makedirs: True - - force: True - - user: edxapp - - group: www-data - - require: - - cmd: run_ansible - -{# Steps to enable git export for courses #} -make_git_export_directory: - file.directory: - - name: {{ git_export_path }} - - user: www-data - - group: www-data - - makedirs: True - -add_private_ssh_key_to_www-data_for_git_export: - file.managed: - - name: /var/www/.ssh/id_rsa - - contents_pillar: edx:ssh_key - - mode: 0600 - - makedirs: True - - dir_mode: 0700 - - user: www-data - - group: www-data - -{% endif %} -{# END states that do not apply to "sandbox" and "devstack" ... #} - -{% if theme_name %} -install_edxapp_theme: - file.directory: - - name: {{ theme_dir }} - - makedirs: True - - user: edxapp - - group: edxapp - git.latest: - - name: {{ theme_repo }} - - branch: {{ theme_branch }} - - rev: {{ theme_branch }} - - target: {{ theme_dir }}/{{ theme_name }} - - user: edxapp - - force_checkout: True - - force_clone: True - - force_reset: True - - force_fetch: True - - update_head: True - - require: - - file: install_edxapp_theme - - require_in: - - cmd: run_ansible -{% endif %} - -{% for host in git_servers %} -add_{{ host.name }}_to_known_hosts_for_edxapp: - ssh_known_hosts.present: - - name: {{ host.name }} - - user: www-data - - enc: ssh-rsa - - fingerprint: "{{ host.fingerprint }}" - - fingerprint_hash_type: sha256 -{% endfor %} diff --git a/salt/edx/refresh_saml_provider_metadata.sls b/salt/edx/refresh_saml_provider_metadata.sls deleted file mode 100644 index 62810de6c..000000000 --- a/salt/edx/refresh_saml_provider_metadata.sls +++ /dev/null @@ -1,5 +0,0 @@ -run_saml_pull: - cmd.run: - - name: '/edx/bin/python.edxapp ./manage.py lms saml --pull --settings=production' - - cwd: /edx/app/edxapp/edx-platform - - runas: www-data diff --git a/salt/edx/templates/extra_locations_lms.j2 b/salt/edx/templates/extra_locations_lms.j2 deleted file mode 100644 index efe543e48..000000000 --- a/salt/edx/templates/extra_locations_lms.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{% raw %} -{% if EDXAPP_SCORM_PKG_STORAGE_DIR %} - location ~ ^/{{ EDXAPP_MEDIA_URL }}/{{ EDXAPP_SCORM_PKG_STORAGE_DIR }}/(?P.*) { - add_header 'Access-Control-Allow-Origin' $cors_origin; - add_header 'Access-Control-Allow-Credentials' 'true'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - - root {{ edxapp_media_dir }}/{{ EDXAPP_SCORM_PKG_STORAGE_DIR }}; - try_files /$file =404; - expires 604800s; - } -{% endif %} -{% endraw %} - -{% set environment = salt.grains.get('environment') %} -{% if environment.startswith('mitxpro') %} - {% set token = salt.pillar.get('edx:mitxpro:registration_access_token') %} - location /register { - return 301 /login; - } - location /user_api/v1/account/registration { - if ($http_x_access_token != "{{ token }}") { - return 403; - } - try_files $uri @proxy_to_lms_app; - } -{% endif %} - -location ~ .*\.php { - return 404; -} - -location ~ ^/wp-(admin|content) { - return 404; -} diff --git a/salt/edx/templates/gitreload_config.json.j2 b/salt/edx/templates/gitreload_config.json.j2 deleted file mode 100644 index 925417242..000000000 --- a/salt/edx/templates/gitreload_config.json.j2 +++ /dev/null @@ -1,11 +0,0 @@ -{ - "PORT": "{{ gr_env.PORT }}", - "UPDATE_LMS": {{ gr_env.UPDATE_LMS }}, - "REPODIR": "{{ gr_env.REPODIR }}", - "LOG_LEVEL": "{{ gr_env.LOG_LEVEL }}", - "NUM_THREADS": {{ gr_env.WORKERS }}, - "LOGFILE": "{{ gr_env.LOGFILE }}", - "VIRTUAL_ENV": "{{ gr_env.VIRTUAL_ENV }}", - "EDX_PLATFORM": "{{ gr_env.EDX_PLATFORM }}", - "DJANGO_SETTINGS": "{{ gr_env.DJANGO_SETTINGS }}" -} diff --git a/salt/edx/templates/gitreload_import.sh.j2 b/salt/edx/templates/gitreload_import.sh.j2 deleted file mode 100644 index c37d3b897..000000000 --- a/salt/edx/templates/gitreload_import.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ -executable=/bin/bash -cd {{ gr_env.EDX_PLATFORM }}/../; -. edxapp_env; -source {{ gr_env.VIRTUAL_ENV }}/bin/activate; -cd {{ gr_env.EDX_PLATFORM }}; -SERVICE_VARIANT=lms python manage.py lms --settings=aws git_add_course {{ item.url }} {{ gr_env.REPODIR }}/{{ item.name }} diff --git a/salt/edx/templates/gitreload_init.conf.j2 b/salt/edx/templates/gitreload_init.conf.j2 deleted file mode 100644 index 84a6981c3..000000000 --- a/salt/edx/templates/gitreload_init.conf.j2 +++ /dev/null @@ -1,21 +0,0 @@ -# gunicorn - -description "gunicorn server" -author "Brandon DeRosier " - -start on (local-filesystems and net-device-up IFACE!=lo) -stop on [!12345] - -respawn -respawn limit 3 30 - -env PID=/var/tmp/gitreload.pid -env WORKERS=1 -env PORT={{ gr_env.PORT }} -env LANG=en_US.UTF-8 -env LOGFILE={{ gr_env.LOGFILE }} - -chdir {{ gr_dir }} -setuid www-data - -exec {{ gr_env.VIRTUAL_ENV }}/bin/gunicorn --preload -b 0.0.0.0:$PORT -w $WORKERS --timeout=10 gitreload.web:app 2>> $LOGFILE >> $LOGFILE diff --git a/salt/edx/templates/gitreload_site.j2 b/salt/edx/templates/gitreload_site.j2 deleted file mode 100644 index c58bfc787..000000000 --- a/salt/edx/templates/gitreload_site.j2 +++ /dev/null @@ -1,58 +0,0 @@ -{% - set key_path = '{}/{}'.format( - salt.pillar.get('edx:edxapp:TLS_LOCATION'), - salt.pillar.get('edx:edxapp:TLS_KEY_NAME') - ) -%} -upstream gitreload_app_server { - ip_hash; - # For a TCP configuration: - server 127.0.0.1:{{ gr_env.PORT }} fail_timeout=300; -} - -server { - listen 443; - server_name {{ hostname }}; - - # MIT SSL Configuration - ssl on; - - ssl_certificate {{ key_path }}.crt; - ssl_certificate_key {{ key_path }}.key; - - ssl_session_timeout 5m; - - ssl_protocols TLSv1.2; - ssl_ciphers TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256; - ssl_prefer_server_ciphers on; - - client_max_body_size 20m; - - auth_basic "Restricted Access"; - auth_basic_user_file {{ htpasswd }}; - - - location / { - try_files $uri @proxy_to_app; - } - - - location @proxy_to_app { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Port $http_x_forwarded_port; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_read_timeout 5m; - - # MITx headers - proxy_set_header HostIP $proxy_add_x_forwarded_for; - - proxy_redirect off; - proxy_pass http://gitreload_app_server; - } - - error_page 502 /502.html; - location = /502.html { - root /etc/nginx/status; - } -} diff --git a/salt/edx/templates/gitreload_systemd.conf.j2 b/salt/edx/templates/gitreload_systemd.conf.j2 deleted file mode 100644 index 685678e44..000000000 --- a/salt/edx/templates/gitreload_systemd.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=gunicorn server running gitreload -Requires=network-online.target -StartLimitInterval=60 -StartLimitBurst=3 - -[Service] -{%- for var, val in gr_env.items() %} -Environment={{ var }}={{ val }} -{% endfor -%} -Environment=LANG=en_US.UTF-8 -Environment=WORKERS=1 -Environment=PID=/var/tmp/gitreload.pid -WorkingDirectory={{ gr_dir }} -User=www-data -ExecStart={{ gr_env.VIRTUAL_ENV }}/bin/gunicorn --preload -b 0.0.0.0:${PORT} -w ${WORKERS} --timeout=10 gitreload.web:app -Restart=always \ No newline at end of file diff --git a/salt/edx/templates/nginx_static_assets.j2 b/salt/edx/templates/nginx_static_assets.j2 deleted file mode 100644 index a0248a6b7..000000000 --- a/salt/edx/templates/nginx_static_assets.j2 +++ /dev/null @@ -1,27 +0,0 @@ - location ~ /static/((?P[^/]+)/(?P.*)|(?P[\w\d\-.]+)) { - add_header Access-Control-Allow-Origin *; - root {% raw %} {{ edxapp_data_dir }} {% endraw %}; - try_files /staticfiles/$dir/$file /course_static/$dir/static/$file /staticfiles/$rfile =404; - - # return a 403 for static files that shouldn't be - # in the staticfiles directory - location ~ ^/static/(?:.*)(?:\.xml|\.json|README.TXT) { - return 403; - } - - location ~ "/static/(?P.*\.[0-9a-f]{12}\.(eot|otf|ttf|woff|woff2)$)" { - add_header Access-Control-Allow-Origin *; - try_files /staticfiles/$collected /course_static/$collected =404; - } - - # Set django-pipelined files to maximum cache time - location ~ "/static/(?P.*\.[0-9a-f]{12}\..*)" { - expires max; - # Without this try_files, files that have been run through - # django-pipeline return 404s - try_files /staticfiles/$collected /course_static/$collected =404; - } - - # Expire other static files immediately (there should be very few / none of these) - expires epoch; - } diff --git a/salt/orchestrate/aws/cloud_profiles/odl-video-service.conf b/salt/orchestrate/aws/cloud_profiles/odl-video-service.conf deleted file mode 100644 index 0ef74d352..000000000 --- a/salt/orchestrate/aws/cloud_profiles/odl-video-service.conf +++ /dev/null @@ -1,18 +0,0 @@ -# -*- mode: yaml; coding: utf-8; -*- -odl-video-service: - provider: mitx - size: t3a.medium - image: {{ salt.sdb.get('sdb://consul/debian_ami_id')|default('ami-0f9e7e8867f55fd8e', True) }} - ssh_username: admin - ssh_interface: private_ips - block_device_mappings: - - DeviceName: {{ salt.sdb.get('sdb://consul/debian_root_device')|default('/dev/xvda', True) }} - Ebs.VolumeSize: 20 - Ebs.VolumeType: gp2 - Ebs.Encrypted: true - iam_profile: odl-video-service-instance-role - tag: - role: odl-video-service - grains: - roles: - - odl-video-service diff --git a/salt/orchestrate/aws/cloud_profiles/starcellbio.conf b/salt/orchestrate/aws/cloud_profiles/starcellbio.conf deleted file mode 100644 index 76fa6e1fa..000000000 --- a/salt/orchestrate/aws/cloud_profiles/starcellbio.conf +++ /dev/null @@ -1,13 +0,0 @@ -# -*- mode: yaml; coding: utf-8; -*- -starcellbio: - provider: mitx - size: t3a.medium - image: {{ salt.sdb.get('sdb://consul/debian_ami_id')|default('ami-0f9e7e8867f55fd8e', True) }} - ssh_username: admin - ssh_interface: private_ips - iam_profile: starcellbio-instance-role - tag: - role: starcellbio - grains: - roles: - - starcellbio diff --git a/salt/orchestrate/aws/odl-video/s3_bucket_policy.sls b/salt/orchestrate/aws/odl-video/s3_bucket_policy.sls deleted file mode 100644 index b9e1f1daf..000000000 --- a/salt/orchestrate/aws/odl-video/s3_bucket_policy.sls +++ /dev/null @@ -1,20 +0,0 @@ -{% set odl_video_bucket_prefix = 'odl-video-service' %} -{% set odl_video_bucket_suffix = salt.environ.get('BUCKET_ENVIRONMENT_SUFFIX', 'rc') %} -{% set odl_video_bucket_purposes = ['dist', 'thumbnails', 'transcoded', 'subtitles'] %} -{% set cloudfront_OriginAccessIdentity = salt.boto_cloudfront.get_distribution('{}-{}'.format(odl_video_bucket_prefix, odl_video_bucket_suffix) - )['result']['distribution']['DistributionConfig']['Origins']['Items'][0]['S3OriginConfig']['OriginAccessIdentity'].split('/')[-1] %} -{% for bucket_purpose in odl_video_bucket_purposes %} -put_{{ bucket_prefix}}-{{ bucket_purpose }}-{{ bucket_suffix }}_policy: - module.run: - boto_s3_bucket.put_policy: - - Bucket: {{ bucket_prefix }}-{{ bucket_purpose }}-{{ bucket_suffix }} - - Policy: - Version: "2008-10-17" - Statement: - - Sid: 1 - Effect: "Allow" - Principal: - AWS: "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity {{ cloudfront_OriginAccessIdentity }}" - Action: "s3:GetObject" - Resource: "arn:aws:s3:::{{ bucket_prefix}}-{{ bucket_purpose }}-{{ bucket_suffix }}/*" -{% endfor %} diff --git a/salt/orchestrate/aws/odl-video/s3_buckets_cloudfront.sls b/salt/orchestrate/aws/odl-video/s3_buckets_cloudfront.sls deleted file mode 100644 index 067015407..000000000 --- a/salt/orchestrate/aws/odl-video/s3_buckets_cloudfront.sls +++ /dev/null @@ -1,113 +0,0 @@ -{% set odl_video_bucket_prefix = 'odl-video-service' %} -{% set odl_video_bucket_suffix = salt.environ.get('BUCKET_ENVIRONMENT_SUFFIX', 'rc') %} -{% set odl_video_bucket_purposes = ['dist', 'thumbnails', 'transcoded', 'subtitles', 'uswitch', 'watch'] %} - -{% for bucket_purpose in odl_video_bucket_purposes %} -create_{{ odl_video_bucket_prefix }}-{{ bucket_purpose }}-{{ bucket_suffix }}: - boto_s3_bucket.present: - - Bucket: {{ odl_video_bucket_prefix }}-{{ bucket_purpose }}-{{ bucket_suffix }} - - region: us-east-1 - - CORSRules: - - AllowedOrigin: ["*"] - AllowedMethod: ["GET"] - AllowedHeader: ["Authorization"] - MaxAgeSconds: 3000 - - Versioning: - Status: "Enabled" -{% endfor %} - -create_cloudfront_distribution_{{ odl_video_bucket_prefix }}-{{ odl_video_bucket_suffix }}: - boto_cloudfront.present: - - name: {{ odl_video_bucket_prefix }}-{{ odl_video_bucket_suffix }} - - config: - CacheBehaviors: - Items: - {% for odl_video_bucket_purpose in ['thumbnails', 'transcoded', 'subtitles'] %} - - AllowedMethods: - CachedMethods: - Items: - - HEAD - - GET - - OPTIONS - Items: - - HEAD - - GET - - OPTIONS - Compress: false - DefaultTTL: 86400 - ForwardedValues: - Cookies: - Forward: none - Headers: - Items: - - Access-Control-Request-Headers - - Access-Control-Request-Method - - Origin - QueryString: false - MaxTTL: 31536000 - MinTTL: 0 - PathPattern: /{{ odl_video_bucket_purpose }}-{{ odl_video_bucket_suffix }}* - SmoothStreaming: false - TargetOriginId: S3-{{ odl_video_bucket_prefix }}-{{ odl_video_bucket_purpose }}-{{ odl_video_bucket_suffix }} - TrustedSigners: - Enabled: false - ViewerProtocolPolicy: redirect-to-https - {% endfor %} - DefaultCacheBehavior: - AllowedMethods: - CachedMethods: - Items: - - HEAD - - GET - - OPTIONS - Items: - - HEAD - - GET - - OPTIONS - Compress: false - DefaultTTL: 86400 - ForwardedValues: - Cookies: - Forward: none - Headers: - Items: - - Access-Control-Request-Headers - - Access-Control-Request-Method - - Origin - QueryString: false - MaxTTL: 31536000 - MinTTL: 0 - SmoothStreaming: false - TargetOriginId: S3-{{ odl_video_bucket_prefix }}-dist-{{ odl_video_bucket_suffix }} - TrustedSigners: - Enabled: true - Items: - - self - ViewerProtocolPolicy: redirect-to-https - DefaultRootObject: '' - Enabled: true - HttpVersion: http2 - IsIPV6Enabled: true - Logging: - Bucket: '' - Enabled: false - IncludeCookies: false - Prefix: '' - Origins: - Items: - {% for odl_video_bucket_purpose in ['dist', 'thumbnails', 'transcoded', 'subtitles'] %} - - CustomHeaders: - DomainName: {{ odl_video_bucket_prefix }}-{{ odl_video_bucket_purpose }}-{{ odl_video_bucket_suffix }}.s3.amazonaws.com - Id: S3-{{ odl_video_bucket_prefix }}-{{ odl_video_bucket_purpose }}-{{ odl_video_bucket_suffix }} - OriginPath: '' - {% endfor %} - PriceClass: PriceClass_All - Restrictions: - GeoRestriction: - RestrictionType: none - ViewerCertificate: - CertificateSource: cloudfront - CloudFrontDefaultCertificate: true - MinimumProtocolVersion: TLSv1.2 - WebACLId: '' - - tags: { 'Name': '{{ odl_video_bucket_prefix }}-{{ odl_video_bucket_suffix }}' } diff --git a/salt/orchestrate/aws/s3_buckets/starcellbio.sls b/salt/orchestrate/aws/s3_buckets/starcellbio.sls deleted file mode 100644 index 85a9be7a6..000000000 --- a/salt/orchestrate/aws/s3_buckets/starcellbio.sls +++ /dev/null @@ -1,15 +0,0 @@ -{% for env in ['rc-apps', 'production-apps'] %} -scb-{{ env }}-microscopy-uploads: - boto_s3_bucket.present: - - Bucket: scb-{{ env }}-microscopy-uploads - - Versioning: - Status: Enabled - - ACL: - - public-read - - region: us-east-1 - - Tagging: - OU: starteam - business_unit: starteam - Department: starteam - Environment: {{ env }} -{% endfor %} diff --git a/salt/reactors/vault/alert_cache_read_misses.sls b/salt/reactors/vault/alert_cache_read_misses.sls deleted file mode 100644 index db9d73173..000000000 --- a/salt/reactors/vault/alert_cache_read_misses.sls +++ /dev/null @@ -1,12 +0,0 @@ -alert_on_cache_read_misses: - local.slack.post_message: - - tgt: 'roles:master' - - tgt_type: grain - - kwarg: - channel: "#devops" - message: | - <@tmacey> <@shaidar> Vault cached read miss on `{{ data['data']['id'] }}`. - ``` - {{ data['data']|json()|indent(10) }} - ``` - from_name: "saltbot" diff --git a/salt/reactors/vault/alert_expiring_leases.sls b/salt/reactors/vault/alert_expiring_leases.sls deleted file mode 100644 index b77111d26..000000000 --- a/salt/reactors/vault/alert_expiring_leases.sls +++ /dev/null @@ -1,12 +0,0 @@ -alert_on_lease_near_expiration: - local.slack.post_message: - - tgt: 'roles:master' - - tgt_type: grain - - kwarg: - channel: "#devops" - message: | - <@tmacey> <@shaidar> The Vault lease `{{ data['data']['id'] }}` will be expiring at `{{ data['data']['expire_time'] }}`. - ``` - {{ data['data']|json()|indent(10) }} - ``` - from_name: "saltbot" diff --git a/salt/reactors/vault/cache_cleanup_on_terminate.sls b/salt/reactors/vault/cache_cleanup_on_terminate.sls deleted file mode 100644 index ae2f53c7a..000000000 --- a/salt/reactors/vault/cache_cleanup_on_terminate.sls +++ /dev/null @@ -1,6 +0,0 @@ -purge_vault_cache_for_terminated_instance: - local.vault.purge_cache_data: - - tgt: 'roles:master' - - tgt_type: grain - - kwarg: - prefix: {{ data['name'] }} diff --git a/salt/top.sls b/salt/top.sls index 161e38dca..ddc8e78c5 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -38,14 +38,7 @@ base: 'roles:cassandra': - match: grain - cassandra - starcellbio*: - - consul - - python - - node - - django - - uwsgi - - nginx - 'G@roles:odl-video-service or G@roles:mitx-cas': + 'G@roles:mitx-cas': - match: compound - utils.configure_debian_source_repos - consul @@ -55,10 +48,6 @@ base: - django - uwsgi - vector - 'roles:odl-video-service': - - match: grain - - utils.logrotate - - vector 'roles:ocw-origin': - match: grain - utils.configure_debian_source_repos diff --git a/salt/vault/roles.sls b/salt/vault/roles.sls deleted file mode 100644 index 62b7a8c09..000000000 --- a/salt/vault/roles.sls +++ /dev/null @@ -1,13 +0,0 @@ -{% set roles = salt.pillar.get('vault:roles') %} -{% for role_id, role in roles.items() %} -create_{{ role_id }}: - vault.role_present: - - name: {{ role.name }} - - mount_point: {{ role.backend }} - - override: {{ salt.pillar.get('vault:force_roles', False) }} - - options: - {% for key, value in role.options.items() %} - {{ key }}: >- - {{ value }} - {% endfor %} -{% endfor %} diff --git a/salt/vault/secret_backends.sls b/salt/vault/secret_backends.sls deleted file mode 100644 index 24e871d9a..000000000 --- a/salt/vault/secret_backends.sls +++ /dev/null @@ -1,42 +0,0 @@ -{% set SIX_MONTHS = '4368h' %} -{% set pki_ttl = '8760h' %} # ONE_YEAR -{% set env_settings = salt.cp.get_url("https://raw.githubusercontent.com/mitodl/salt-ops/main/salt/environment_settings.yml", dest=None)|load_yaml %} - -enable_transit_secret_backend: - vault.secret_backend_enabled: - - backend_type: transit - - description: Backend to provide encryption, hashing, and randomness as a service - -enable_mitx_aws_secret_backend: - vault.secret_backend_enabled: - - backend_type: aws - - mount_point: aws-mitx - - description: Backend to dynamically create IAM credentials - - ttl_max: {{ SIX_MONTHS }} - - ttl_default: {{ SIX_MONTHS }} - - lease_max: {{ SIX_MONTHS }} - - lease_default: {{ SIX_MONTHS }} - -enable_pki_intermediate_backend: - vault.secret_backend_enabled: - - backend_type: pki - - mount_point: pki-intermediate-ca - - description: Backend to create certificates signed by our root CA - - ttl_default: {{ pki_ttl }} - -{% for env_name in env_settings.environments %} -enable_pki_intermediate_{{ env_name }}_backend: - vault.secret_backend_enabled: - - backend_type: pki - - mount_point: pki-intermediate-{{ env_name }} - - description: Backend to create certificates for {{ env_name }} - - ttl_default: {{ pki_ttl }} -{% endfor %} - -{% for unit in salt.pillar.get('business_units', []) %} -enable_generic_backend_for_{{ unit }}: - vault.secret_backend_enabled: - - backend_type: generic - - mount_point: secret-{{ unit }} - - description: Secrets storage for values pertaining to {{ unit }} -{% endfor %}