Skip to content

Commit

Permalink
chore: Updating Python Requirements
Browse files Browse the repository at this point in the history
  • Loading branch information
@edx-requirements-bot @mumarkhan999
edx-requirements-bot authored and mumarkhan999 committed Jan 7, 2025
1 parent a2b493d commit 6bd71d1
Show file tree
Hide file tree
Showing 5 changed files with 169 additions and 100 deletions.
9 changes: 2 additions & 7 deletions .pylintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,6 @@ disable=
# (visual studio) and html
output-format=text

# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no

# Tells whether to display a full report or only the messages
reports=no

Expand Down Expand Up @@ -120,7 +115,7 @@ generated-members=
[BASIC]

# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input
bad-names=map,filter,apply,input

# Regular expression which should only match correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
Expand Down Expand Up @@ -240,7 +235,7 @@ max-locals=15
max-returns=6

# Maximum number of branch for function / method body
max-branchs=12
max-branches=12

# Maximum number of statements in function / method body
max-statements=50
Expand Down
4 changes: 2 additions & 2 deletions requirements/base.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
#
boto==2.49.0
# via google-compute-engine
boto3==1.35.73
boto3==1.35.74
# via -r requirements/base.in
botocore==1.35.73
botocore==1.35.74
# via
# -r requirements/base.in
# boto3
Expand Down
16 changes: 7 additions & 9 deletions requirements/test.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#
annotated-types==0.7.0
# via pydantic
astroid==3.2.4
astroid==3.3.5
# via pylint
attrs==24.2.0
# via
Expand All @@ -20,12 +20,12 @@ boto==2.49.0
# via
# -r requirements/base.txt
# google-compute-engine
boto3==1.35.73
boto3==1.35.74
# via
# -r requirements/base.txt
# aws-sam-translator
# moto
botocore==1.35.73
botocore==1.35.74
# via
# -r requirements/base.txt
# aws-xray-sdk
Expand Down Expand Up @@ -129,14 +129,12 @@ pycparser==2.22
# via cffi
pycurl==7.45.3
# via pyresttest
pydantic==2.10.2
pydantic==2.10.3
# via aws-sam-translator
pydantic-core==2.27.1
# via pydantic
pylint==3.2.6
# via
# -c requirements/constraints.txt
# -r requirements/test.in
pylint==3.3.2
# via -r requirements/test.in
pyparsing==3.2.0
# via moto
pyresttest==1.7.1
Expand Down Expand Up @@ -173,7 +171,7 @@ responses==0.25.3
# via moto
rfc3339-validator==0.1.4
# via openapi-schema-validator
rpds-py==0.22.0
rpds-py==0.22.1
# via
# jsonschema
# referencing
Expand Down
114 changes: 76 additions & 38 deletions scripts/aws/deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,44 +13,82 @@
parser = argparse.ArgumentParser()

parser.add_argument("--aws-region", required=False, default="us-east-1")
parser.add_argument("--api-base-domain", required=True,
help="The name of the API Gateway domain to be created.")
parser.add_argument("--swagger-filename", required=True,
help="The name of a complete Swagger 2.0 specification file with AWS vendor hooks.")
parser.add_argument("--tag", required=True,
help="Unique identifier for this deployment (such as a git hash)")
parser.add_argument("--rotation-order", required=True, nargs='+',
help="Ordered list of stages in the deployment ring (ex: 'red black')")
parser.add_argument("--log-level", required=False, default="OFF", choices=['OFF', 'ERROR', 'INFO'],
help="Verbosity of messages sent to CloudWatch Logs")
parser.add_argument("--metrics", required=False, default="false", choices=['false', 'true'],
help="Enable CloudWatch metrics")
parser.add_argument("--caching", required=False, default="false", choices=['false', 'true'],
help="Enable API Gateway caching feature")
parser.add_argument("--rate-limit", required=False, default="500", type=str,
help="Default per-resource average rate limit")
parser.add_argument("--burst-limit", required=False, default="1000", type=str,
help="Default per-resource maximum rate limit")
parser.add_argument("--landing-page", required=True,
help="Location of landing page for 'root' level requests")
parser.add_argument("--edxapp-host", required=True,
help="Location of edxapp for request routing")
parser.add_argument("--catalog-host", required=True,
help="Location of catalog IDA for request routing")
parser.add_argument("--enterprise-host", required=False, default='',
help="Location of enterprise IDA for request routing")
parser.add_argument('--analytics-api-host', required=True,
help="Location of analyitcs-api IDA for request routing")
parser.add_argument('--registrar-host', required=True,
help="Location of registrar IDA for request routing")
parser.add_argument('--enterprise-catalog-host', required=True,
help="Location of enterprise catalog IDA for request routing")
parser.add_argument('--authoring-host', required=True,
help="Location of Studio for authoring request routing")
parser.add_argument('--license-manager-host', required=True,
help="Location of License Manager IDA for request routing")
parser.add_argument('--enterprise-access-host', required=True,
help="Location of Enterprise Access IDA for request routing")
parser.add_argument(
"--api-base-domain", required=True,
help="The name of the API Gateway domain to be created."
)
parser.add_argument(
"--swagger-filename", required=True,
help="The name of a complete Swagger 2.0 specification file with AWS vendor hooks."
)
parser.add_argument(
"--tag", required=True,
help="Unique identifier for this deployment (such as a git hash)"
)
parser.add_argument(
"--rotation-order", required=True, nargs='+',
help="Ordered list of stages in the deployment ring (ex: 'red black')"
)
parser.add_argument(
"--log-level", required=False, default="OFF", choices=['OFF', 'ERROR', 'INFO'],
help="Verbosity of messages sent to CloudWatch Logs"
)
parser.add_argument(
"--metrics", required=False, default="false", choices=['false', 'true'],
help="Enable CloudWatch metrics"
)
parser.add_argument(
"--caching", required=False, default="false", choices=['false', 'true'],
help="Enable API Gateway caching feature"
)
parser.add_argument(
"--rate-limit", required=False, default="500", type=str,
help="Default per-resource average rate limit"
)
parser.add_argument(
"--burst-limit", required=False, default="1000", type=str,
help="Default per-resource maximum rate limit"
)
parser.add_argument(
"--landing-page", required=True,
help="Location of landing page for 'root' level requests"
)
parser.add_argument(
"--edxapp-host", required=True,
help="Location of edxapp for request routing"
)
parser.add_argument(
"--catalog-host", required=True,
help="Location of catalog IDA for request routing"
)
parser.add_argument(
"--enterprise-host", required=False, default='',
help="Location of enterprise IDA for request routing"
)
parser.add_argument(
'--analytics-api-host', required=True,
help="Location of analyitcs-api IDA for request routing"
)
parser.add_argument(
'--registrar-host', required=True,
help="Location of registrar IDA for request routing"
)
parser.add_argument(
'--enterprise-catalog-host', required=True,
help="Location of enterprise catalog IDA for request routing"
)
parser.add_argument(
'--authoring-host', required=True,
help="Location of Studio for authoring request routing"
)
parser.add_argument(
'--license-manager-host', required=True,
help="Location of License Manager IDA for request routing"
)
parser.add_argument(
'--enterprise-access-host', required=True,
help="Location of Enterprise Access IDA for request routing"
)

cli_args = parser.parse_args()
integration_settings = {
Expand Down
126 changes: 82 additions & 44 deletions scripts/aws/monitor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def get_api_id(client, api_base_domain):
return (response['restApiId'], response['stage'])


def create_api_alarm(cw_session, alarm_name, metric,
def create_api_alarm(*, cw_session, alarm_name, metric,
namespace, stat, comparison, description,
threshold, period, eval_period, dimensions, topic):
"""Puts data to the metric, then creates the alarm for appropriate metric in API Gateway"""
Expand Down Expand Up @@ -92,7 +92,7 @@ def create_lambda_function_zip(jinja_env, temp_dir, splunk_host, splunk_token, l
return zip_file


def get_lambda_exec_policy(jinja_env, temp_dir, region, acct_id, func_name, kms_key):
def get_lambda_exec_policy(*, jinja_env, temp_dir, region, acct_id, func_name, kms_key):
"""updates the policy json and returns it"""
resource_values = {
'region': region,
Expand Down Expand Up @@ -165,7 +165,7 @@ def create_role_with_managed_policy(iam, role_name, assume_role_policy_document,
return response['Role']['Arn']


def create_lambda_function(client, function_name, runtime, role,
def create_lambda_function(*, client, function_name, runtime, role,
handler, zip_file, description, timeout, mem_size, vpc):
"""Creates a lambda function to pull data from cloudwatch event.
It only works works in VPC"""
Expand Down Expand Up @@ -274,42 +274,71 @@ def add_cloudwatchlog_role_to_apigateway(client, role_arn):
sns_client = session.create_client('sns', args.aws_region)
cw = session.create_client('cloudwatch', args.aws_region)

create_api_alarm(cw, 'api-gateway-count', 'Count', 'ApiGateway',
'Average', 'GreaterThanOrEqualToThreshold',
'Average API count for a period of 5 min', 50, 300, 1,
[{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}],
get_topic_arn(sns_client, 'aws-non-critical-alert'))

create_api_alarm(cw, 'api-gateway-latency', 'Latency', 'ApiGateway', 'Average',
'GreaterThanOrEqualToThreshold', 'Average API Latency for a period of 5 min', 3, 300, 1,
[{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}],
get_topic_arn(sns_client, 'aws-non-critical-alert'))

create_api_alarm(cw, 'api-gateway-errors-4xx', '4XXError', 'ApiGateway', 'Average',
'GreaterThanOrEqualToThreshold', 'Average 4XX errors for a period of 5 min', 4, 300, 1,
[{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}],
get_topic_arn(sns_client, 'aws-non-critical-alert'))

create_api_alarm(cw, 'api-gateway-errors-5xx', '5XXError', 'ApiGateway', 'Average',
'GreaterThanOrEqualToThreshold', 'Average 5XX errors for a period of 5 min', 4, 300, 1,
[{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage}, {'Name': 'ApiId', 'Value': api_id}],
get_topic_arn(sns_client, 'aws-non-critical-alert'))

lambda_exec_role_arn = create_role_with_inline_policy(iam_client, lambda_role_name,
'{"Version": "2012-10-17","Statement": '
'[{"Effect": "Allow","Principal": '
'{"Service": "lambda.amazonaws.com"},'
'"Action": "sts:AssumeRole"}]}',
open(get_lambda_exec_policy(j2_env, tmpdirname, # pylint: disable=consider-using-with
args.aws_region,
args.acct_id,
lambda_function_name,
args.kms_key),
encoding='utf-8').read())
create_api_alarm(
cw_session=cw, alarm_name='api-gateway-count', metric='Count', namespace='ApiGateway', stat='Average',
comparison='GreaterThanOrEqualToThreshold', description='Average API count for a period of 5 min',
threshold=50, period=300, eval_period=1,
dimensions=[
{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage},
{'Name': 'ApiId', 'Value': api_id}
],
topic=get_topic_arn(sns_client, 'aws-non-critical-alert')
)

create_api_alarm(
cw_session=cw, alarm_name='api-gateway-latency', metric='Latency', namespace='ApiGateway', stat='Average',
comparison='GreaterThanOrEqualToThreshold', description='Average API Latency for a period of 5 min',
threshold=3, period=300, eval_period=1,
dimensions=[
{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage},
{'Name': 'ApiId', 'Value': api_id}
],
topic=get_topic_arn(sns_client, 'aws-non-critical-alert')
)

create_api_alarm(
cw_session=cw, alarm_name='api-gateway-errors-4xx', metric='4XXError', namespace='ApiGateway', stat='Average',
comparison='GreaterThanOrEqualToThreshold', description='Average 4XX errors for a period of 5 min',
threshold=4, period=300, eval_period=1,
dimensions=[
{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage},
{'Name': 'ApiId', 'Value': api_id}
],
topic=get_topic_arn(sns_client, 'aws-non-critical-alert')
)

create_api_alarm(
cw_session=cw, alarm_name='api-gateway-errors-5xx', metric='5XXError', namespace='ApiGateway', stat='Average',
comparison='GreaterThanOrEqualToThreshold', description='Average 5XX errors for a period of 5 min',
threshold=4, period=300, eval_period=1,
dimensions=[
{'Name': 'ApiName', 'Value': api_gateway_name},
{'Name': 'Stage', 'Value': api_stage},
{'Name': 'ApiId', 'Value': api_id}
],
topic=get_topic_arn(sns_client, 'aws-non-critical-alert')
)

lambda_exec_role_arn = create_role_with_inline_policy(
iam_client,
lambda_role_name,
'{"Version": "2012-10-17","Statement": [{"Effect": "Allow","Principal": {"Service": "lambda.amazonaws.com"},'
'"Action": "sts:AssumeRole"}]}',
open(
get_lambda_exec_policy(
jinja_env=j2_env,
temp_dir=tmpdirname, # pylint: disable=consider-using-with
region=args.aws_region,
acct_id=args.acct_id,
func_name=lambda_function_name,
kms_key=args.kms_key
),
encoding='utf-8'
).read()
)

logging.info('Waiting for the newly created role to be available')
# Sleep for 10 seconds to allow the role created above to be avialable for lambda function creation
Expand All @@ -320,11 +349,20 @@ def add_cloudwatchlog_role_to_apigateway(client, role_arn):
zip_file_name = create_lambda_function_zip(j2_env, tmpdirname, args.splunk_host,
args.splunk_token, lambda_function_name)
vpc_config = {'SubnetIds': args.subnet_list, 'SecurityGroupIds': args.sg_list}
create_lambda_function(lambda_client, lambda_function_name, 'nodejs18.x', lambda_exec_role_arn,
'index.handler', zip_file_name,
'Demonstrates logging events to Splunk HTTP Event '
'Collector, accessing resources in a VPC', args.lambda_timeout, args.lambda_memory,
vpc_config)
create_lambda_function(
client=lambda_client,
function_name=lambda_function_name,
runtime='nodejs18.x',
role=lambda_exec_role_arn,
handler='index.handler',
zip_file=zip_file_name,
description=(
'Demonstrates logging events to Splunk HTTP Event Collector, accessing resources in a VPC'
),
timeout=args.lambda_timeout,
mem_size=args.lambda_memory,
vpc=vpc_config
)
try:
shutil.rmtree(tmpdirname)
except OSError as exc:
Expand Down

0 comments on commit 6bd71d1

Please sign in to comment.