-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #753 from JiscSD/OC-925
OC-925: Configure ECS task to run on a schedule
- Loading branch information
Showing
12 changed files
with
159 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,45 +1,11 @@ | ||
| import * as I from 'interface'; | ||
| import * as ingestLogService from 'ingestLog/service'; | ||
| import * as integrationService from 'integration/service'; | ||
| import * as response from 'lib/response'; | ||
|
|
||
| export const incrementalAriIngest = async ( | ||
| event: I.APIRequest | I.EventBridgeEvent<'Scheduled Event', string> | ||
| export const triggerAriIngest = async ( | ||
| event: I.APIRequest<undefined, I.TriggerAriIngestQueryParams, undefined> | ||
| ): Promise<I.JSONResponse> => { | ||
| // Check if a process is currently running. | ||
| const lastLog = await ingestLogService.getMostRecentLog('ARI', true); | ||
| const triggeredByHttp = event && 'headers' in event; | ||
| const dryRun = triggeredByHttp ? !!event.queryStringParameters?.dryRun : false; | ||
| const dryRunMessages: string[] = []; | ||
|
|
||
| if (lastLog && !lastLog.end) { | ||
| if (dryRun) { | ||
| dryRunMessages.push( | ||
| 'This run would have been cancelled because another run is currently in progress. However, the run has still been simulated.' | ||
| ); | ||
| } else { | ||
| return response.json(202, { | ||
| message: 'Cancelling ingest. Either an import is already in progress or the last import failed.' | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| try { | ||
| const ingestResult = await integrationService.incrementalAriIngest(dryRun, 'email'); | ||
|
|
||
| return response.json( | ||
| 200, | ||
| dryRunMessages.length ? { messages: [...dryRunMessages, ingestResult] } : ingestResult | ||
| ); | ||
| } catch (error) { | ||
| console.log(error); | ||
|
|
||
| return response.json(500, { message: 'Unknown server error.' }); | ||
| } | ||
| }; | ||
|
|
||
| export const triggerECSTask = async (): Promise<I.JSONResponse> => { | ||
| const triggerTaskOutput = await integrationService.triggerECSTask(); | ||
| const triggerTaskOutput = await integrationService.triggerAriIngest(event.queryStringParameters.dryRun); | ||
|
|
||
| return response.json(200, { message: triggerTaskOutput }); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| export { default as incrementalAriIngestHttp } from './incrementalAriIngestHttp'; | ||
| export { default as triggerAriIngest } from './triggerAriIngest'; |
5 changes: 3 additions & 2 deletions
5
...ration/schema/incrementalAriIngestHttp.ts → ...ts/integration/schema/triggerAriIngest.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| # Code adapted from https://medium.com/@igorkachmaryk/using-terraform-to-setup-aws-eventbridge-scheduler-and-a-scheduled-ecs-task-1208ae077360 | ||
| resource "aws_iam_role" "scheduler" { | ||
| name = "cron-scheduler-role-${var.environment}-${var.project_name}" | ||
| assume_role_policy = jsonencode({ | ||
| Version = "2012-10-17" | ||
| Statement = [ | ||
| { | ||
| Effect = "Allow" | ||
| Principal = { | ||
| Service = ["scheduler.amazonaws.com"] | ||
| } | ||
| Action = "sts:AssumeRole" | ||
| } | ||
| ] | ||
| }) | ||
| } | ||
|
|
||
| resource "aws_iam_role_policy_attachment" "scheduler" { | ||
| policy_arn = aws_iam_policy.scheduler.arn | ||
| role = aws_iam_role.scheduler.name | ||
| } | ||
|
|
||
| resource "aws_iam_policy" "scheduler" { | ||
| name = "cron-scheduler-policy-${var.environment}-${var.project_name}" | ||
| policy = jsonencode({ | ||
| Version = "2012-10-17" | ||
| Statement = [ | ||
| { | ||
| Effect = "Allow", | ||
| Action = [ | ||
| "ecs:RunTask" | ||
| ] | ||
| # Replace revision number with * | ||
| Resource = ["${trimsuffix(aws_ecs_task_definition.ari-import.arn, ":${aws_ecs_task_definition.ari-import.revision}")}:*"] | ||
| }, | ||
| { | ||
| Effect = "Allow", | ||
| Action = [ | ||
| "iam:PassRole" | ||
| ] | ||
| Resource = [aws_iam_role.ecs-task-role.arn, aws_iam_role.ecs-task-exec-role.arn] | ||
| }, | ||
| { | ||
| Action = [ | ||
| "sqs:SendMessage" | ||
| ], | ||
| Effect = "Allow", | ||
| Resource = [aws_sqs_queue.scheduler-dlq.arn] | ||
| } | ||
| ] | ||
| }) | ||
| } | ||
|
|
||
| resource "aws_scheduler_schedule" "ari_import_cron" { | ||
| name = "ari-import-schedule-int" | ||
|
|
||
| flexible_time_window { | ||
| mode = "OFF" | ||
| } | ||
|
|
||
| schedule_expression = "cron(0 5 ? * TUE *)" # Run every Tuesday at 5AM | ||
|
|
||
| target { | ||
| arn = aws_ecs_cluster.ecs.arn | ||
| role_arn = aws_iam_role.scheduler.arn | ||
|
|
||
| # On prod, override container command to do a dry run instead of a real one. | ||
| # The output will be checked before manually triggering a real run using the API. | ||
| input = terraform.workspace == "prod" ? jsonencode({ | ||
| containerOverrides = [ | ||
| { | ||
| command = ["npm", "run", "ariImport", "--", "dryRun=true", "reportFormat=email"] | ||
| name = "ari-import" | ||
| } | ||
| ] | ||
| }) : null | ||
|
|
||
| dead_letter_config { | ||
| arn = aws_sqs_queue.scheduler-dlq.arn | ||
| } | ||
|
|
||
| ecs_parameters { | ||
| # Trimming the revision suffix here so that schedule always uses latest revision | ||
| task_definition_arn = trimsuffix(aws_ecs_task_definition.ari-import.arn, ":${aws_ecs_task_definition.ari-import.revision}") | ||
| launch_type = "FARGATE" | ||
|
|
||
| network_configuration { | ||
| security_groups = [aws_security_group.ari-import-task-sg.id] | ||
| subnets = var.private_subnet_ids | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| resource "aws_sqs_queue" "scheduler-dlq" { | ||
| name = "scheduler-dlq-${var.environment}-${var.project_name}" | ||
| } |