diff --git a/python/GLUE_CRAWLER_CONNECTION_USES_SSL/glue_crawler_connection_uses_ssl_check.py b/python/GLUE_CRAWLER_CONNECTION_USES_SSL/glue_crawler_connection_uses_ssl_check.py new file mode 100644 index 00000000..1b8d5a64 --- /dev/null +++ b/python/GLUE_CRAWLER_CONNECTION_USES_SSL/glue_crawler_connection_uses_ssl_check.py @@ -0,0 +1,368 @@ +import boto3, sys, datetime, botocore, json, os +try: + import liblogging +except ImportError: + pass + +############## +# Parameters # +############## + +# Define the default resource to report to Config Rules +DEFAULT_RESOURCE_TYPE = 'AWS::Glue::Connection' + +# Set to True to get the lambda to assume the Role attached on the Config Service (useful for cross-account). +ASSUME_ROLE_MODE = False + +# Other parameters (no change needed) +CONFIG_ROLE_TIMEOUT_SECONDS = 900 + +############# +# Main Code # +############# + +def get_all_glue_connections(glue_client): + all_connections = [] + + connections = glue_client.get_connections() + all_connections += connections['ConnectionList'] + + while True: + if 'NextToken' in connections: + connections = glue_client.get_connections(NextToken=connections['NextToken']) + all_connections += connections['ConnectionList'] + else: + break + + return all_connections + +def evaluate_compliance(event, configuration_item, valid_rule_parameters): + glue_client = boto3.client('glue') + evaluations = [] + + all_connections = get_all_glue_connections(glue_client) + + # No log group exists + if not all_connections: + return None + + for connection in all_connections: + if connection['ConnectionType'] == 'JDBC' or connection['ConnectionType'] == 'MONGODB': + #if encryption is not enabled for both then put into non-compliant evaluation + if connection['ConnectionProperties']['JDBC_ENFORCE_SSL'] != 'true': + evaluations.append(build_evaluation(connection['Name'], + 'NON_COMPLIANT', + event, + annotation="This connection doesn't require SSL")) + + else: + evaluations.append(build_evaluation(connection['Name'], + 'COMPLIANT', + event, + annotation="This connection requires SSL")) + print(evaluations[0]) + return evaluations + +def evaluate_parameters(rule_parameters): + if 'KmsKeyId' not in rule_parameters: + return {} + + if 'arn:aws:kms' not in rule_parameters['KmsKeyId']: + raise ValueError('Invalid value for paramter KmsKeyId, Expected KMS Key ARN') + + return rule_parameters + +#################### +# Helper Functions # +#################### + +# Build an error to be displayed in the logs when the parameter is invalid. +def build_parameters_value_error_response(ex): + """Return an error dictionary when the evaluate_parameters() raises a ValueError. + Keyword arguments: + ex -- Exception text + """ + return build_error_response(internal_error_message="Parameter value is invalid", + internal_error_details="An ValueError was raised during the validation of the Parameter value", + customer_error_code="InvalidParameterValueException", + customer_error_message=str(ex)) + +# This gets the client after assuming the Config service role +# either in the same AWS account or cross-account. +def get_client(service, event): + """Return the service boto client. It should be used instead of directly calling the client. + Keyword arguments: + service -- the service name used for calling the boto.client() + event -- the event variable given in the lambda handler + """ + if not ASSUME_ROLE_MODE: + return boto3.client(service) + credentials = get_assume_role_credentials(event["executionRoleArn"]) + return boto3.client(service, aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'] + ) + +# This generate an evaluation for config +def build_evaluation(resource_id, compliance_type, event, resource_type=DEFAULT_RESOURCE_TYPE, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on scheduled rules. + Keyword arguments: + resource_id -- the unique id of the resource to report + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + event -- the event variable given in the lambda handler + resource_type -- the CloudFormation resource type (or AWS::::Account) to report on the rule (default DEFAULT_RESOURCE_TYPE) + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_cc = {} + if annotation: + eval_cc['Annotation'] = annotation + eval_cc['ComplianceResourceType'] = resource_type + eval_cc['ComplianceResourceId'] = resource_id + eval_cc['ComplianceType'] = compliance_type + eval_cc['OrderingTimestamp'] = str(json.loads(event['invokingEvent'])['notificationCreationTime']) + return eval_cc + +def build_evaluation_from_config_item(configuration_item, compliance_type, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on configuration change rules. + Keyword arguments: + configuration_item -- the configurationItem dictionary in the invokingEvent + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_ci = {} + if annotation: + eval_ci['Annotation'] = annotation + eval_ci['ComplianceResourceType'] = configuration_item['resourceType'] + eval_ci['ComplianceResourceId'] = configuration_item['resourceId'] + eval_ci['ComplianceType'] = compliance_type + eval_ci['OrderingTimestamp'] = configuration_item['configurationItemCaptureTime'] + return eval_ci + +#################### +# Boilerplate Code # +#################### + +# Helper function used to validate input +def check_defined(reference, reference_name): + if not reference: + raise Exception('Error: ', reference_name, 'is not defined') + return reference + +# Check whether the message is OversizedConfigurationItemChangeNotification or not +def is_oversized_changed_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'OversizedConfigurationItemChangeNotification' + +# Check whether the message is a ScheduledNotification or not. +def is_scheduled_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'ScheduledNotification' + +# Get configurationItem using getResourceConfigHistory API +# in case of OversizedConfigurationItemChangeNotification +def get_configuration(resource_type, resource_id, configuration_capture_time): + result = AWS_CONFIG_CLIENT.get_resource_config_history( + resourceType=resource_type, + resourceId=resource_id, + laterTime=configuration_capture_time, + limit=1) + configuration_item = result['configurationItems'][0] + return convert_api_configuration(configuration_item) + +# Convert from the API model to the original invocation model +def convert_api_configuration(configuration_item): + for k, v in configuration_item.items(): + if isinstance(v, datetime.datetime): + configuration_item[k] = str(v) + configuration_item['awsAccountId'] = configuration_item['accountId'] + configuration_item['ARN'] = configuration_item['arn'] + configuration_item['configurationStateMd5Hash'] = configuration_item['configurationItemMD5Hash'] + configuration_item['configurationItemVersion'] = configuration_item['version'] + configuration_item['configuration'] = json.loads(configuration_item['configuration']) + if 'relationships' in configuration_item: + for i in range(len(configuration_item['relationships'])): + configuration_item['relationships'][i]['name'] = configuration_item['relationships'][i]['relationshipName'] + return configuration_item + +# Based on the type of message get the configuration item +# either from configurationItem in the invoking event +# or using the getResourceConfigHistiry API in getConfiguration function. +def get_configuration_item(invoking_event): + check_defined(invoking_event, 'invokingEvent') + if is_oversized_changed_notification(invoking_event['messageType']): + configuration_item_summary = check_defined(invoking_event['configuration_item_summary'], 'configurationItemSummary') + return get_configuration(configuration_item_summary['resourceType'], configuration_item_summary['resourceId'], configuration_item_summary['configurationItemCaptureTime']) + if is_scheduled_notification(invoking_event['messageType']): + return None + return check_defined(invoking_event['configurationItem'], 'configurationItem') + +# Check whether the resource has been deleted. If it has, then the evaluation is unnecessary. +def is_applicable(configuration_item, event): + try: + check_defined(configuration_item, 'configurationItem') + check_defined(event, 'event') + except: + return True + status = configuration_item['configurationItemStatus'] + event_left_scope = event['eventLeftScope'] + if status == 'ResourceDeleted': + print("Resource Deleted, setting Compliance Status to NOT_APPLICABLE.") + return status in ('OK', 'ResourceDiscovered') and not event_left_scope + +def get_assume_role_credentials(role_arn): + sts_client = boto3.client('sts') + try: + assume_role_response = sts_client.assume_role(RoleArn=role_arn, + RoleSessionName="configLambdaExecution", + DurationSeconds=CONFIG_ROLE_TIMEOUT_SECONDS) + if 'liblogging' in sys.modules: + liblogging.logSession(role_arn, assume_role_response) + return assume_role_response['Credentials'] + except botocore.exceptions.ClientError as ex: + # Scrub error message for any internal account info leaks + print(str(ex)) + if 'AccessDenied' in ex.response['Error']['Code']: + ex.response['Error']['Message'] = "AWS Config does not have permission to assume the IAM role." + else: + ex.response['Error']['Message'] = "InternalError" + ex.response['Error']['Code'] = "InternalError" + raise ex + +# This removes older evaluation (usually useful for periodic rule not reporting on AWS::::Account). +def clean_up_old_evaluations(latest_evaluations, event): + + cleaned_evaluations = [] + + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100) + + old_eval_list = [] + + while True: + for old_result in old_eval['EvaluationResults']: + old_eval_list.append(old_result) + if 'NextToken' in old_eval: + next_token = old_eval['NextToken'] + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100, + NextToken=next_token) + else: + break + + for old_eval in old_eval_list: + old_resource_id = old_eval['EvaluationResultIdentifier']['EvaluationResultQualifier']['ResourceId'] + newer_founded = False + for latest_eval in latest_evaluations: + if old_resource_id == latest_eval['ComplianceResourceId']: + newer_founded = True + if not newer_founded: + cleaned_evaluations.append(build_evaluation(old_resource_id, "NOT_APPLICABLE", event)) + + return cleaned_evaluations + latest_evaluations + +def lambda_handler(event, context): + if 'liblogging' in sys.modules: + liblogging.logEvent(event) + + global AWS_CONFIG_CLIENT + + #print(event) + check_defined(event, 'event') + invoking_event = json.loads(event['invokingEvent']) + rule_parameters = {} + if 'ruleParameters' in event: + rule_parameters = json.loads(event['ruleParameters']) + + try: + valid_rule_parameters = evaluate_parameters(rule_parameters) + except ValueError as ex: + return build_parameters_value_error_response(ex) + + try: + AWS_CONFIG_CLIENT = get_client('config', event) + if invoking_event['messageType'] in ['ConfigurationItemChangeNotification', 'ScheduledNotification', 'OversizedConfigurationItemChangeNotification']: + configuration_item = get_configuration_item(invoking_event) + if is_applicable(configuration_item, event): + compliance_result = evaluate_compliance(event, configuration_item, valid_rule_parameters) + else: + compliance_result = "NOT_APPLICABLE" + else: + return build_internal_error_response('Unexpected message type', str(invoking_event)) + except botocore.exceptions.ClientError as ex: + if is_internal_error(ex): + return build_internal_error_response("Unexpected error while completing API request", str(ex)) + return build_error_response("Customer error while making API request", str(ex), ex.response['Error']['Code'], ex.response['Error']['Message']) + except ValueError as ex: + return build_internal_error_response(str(ex), str(ex)) + + evaluations = [] + latest_evaluations = [] + + if not compliance_result: + latest_evaluations.append(build_evaluation(event['accountId'], "NOT_APPLICABLE", event, resource_type='AWS::::Account')) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, str): + if configuration_item: + evaluations.append(build_evaluation_from_config_item(configuration_item, compliance_result)) + else: + evaluations.append(build_evaluation(event['accountId'], compliance_result, event, resource_type=DEFAULT_RESOURCE_TYPE)) + elif isinstance(compliance_result, list): + for evaluation in compliance_result: + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in evaluation: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + + if not missing_fields: + latest_evaluations.append(evaluation) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, dict): + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in compliance_result: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + if not missing_fields: + evaluations.append(compliance_result) + else: + evaluations.append(build_evaluation_from_config_item(configuration_item, 'NOT_APPLICABLE')) + + # Put together the request that reports the evaluation status + result_token = event['resultToken'] + test_mode = False + if result_token == 'TESTMODE': + # Used solely for RDK test to skip actual put_evaluation API call + test_mode = True + + # Invoke the Config API to report the result of the evaluation + evaluation_copy = [] + evaluation_copy = evaluations[:] + while evaluation_copy: + AWS_CONFIG_CLIENT.put_evaluations(Evaluations=evaluation_copy[:100], ResultToken=result_token, TestMode=test_mode) + del evaluation_copy[:100] + + # Used solely for RDK test to be able to test Lambda function + return evaluations + +def is_internal_error(exception): + return ((not isinstance(exception, botocore.exceptions.ClientError)) or exception.response['Error']['Code'].startswith('5') + or 'InternalError' in exception.response['Error']['Code'] or 'ServiceError' in exception.response['Error']['Code']) + +def build_internal_error_response(internal_error_message, internal_error_details=None): + return build_error_response(internal_error_message, internal_error_details, 'InternalError', 'InternalError') + +def build_error_response(internal_error_message, internal_error_details=None, customer_error_code=None, customer_error_message=None): + error_response = { + 'internalErrorMessage': internal_error_message, + 'internalErrorDetails': internal_error_details, + 'customerErrorMessage': customer_error_message, + 'customerErrorCode': customer_error_code + } + print(error_response) + return error_response + diff --git a/python/GLUE_CRAWLER_CONNECTION_USES_SSL/remediate_glue_connection_ssl.py b/python/GLUE_CRAWLER_CONNECTION_USES_SSL/remediate_glue_connection_ssl.py new file mode 100644 index 00000000..1a2068ce --- /dev/null +++ b/python/GLUE_CRAWLER_CONNECTION_USES_SSL/remediate_glue_connection_ssl.py @@ -0,0 +1,23 @@ +import boto3, logging, sys +from botocore.exceptions import ClientError + +def lambda_handler(event, context): + print(event) + + connection_name = event['ResourceId'] + glue_client = boto3.client('glue') + + #get connection type because its required for update_connection + response = glue_client.get_connection(Name=connection_name, HidePassword=False) + connection_type = response['Connection']['ConnectionType'] + connection_properties = response['Connection']['ConnectionProperties'] + connection_properties["JDBC_ENFORCE_SSL"] = "true" #overwrite to enfore ssl + + glue_client.update_connection( + Name=connection_name, + ConnectionInput= { + 'Name': connection_name, + 'ConnectionType': connection_type, + 'ConnectionProperties' : connection_properties + } + ) \ No newline at end of file diff --git a/python/GLUE_CRAWLER_SECURITY_CONFIGURATION_ENCRYPTION/glue_crawler_security_configuration_is_encrypted_check.py b/python/GLUE_CRAWLER_SECURITY_CONFIGURATION_ENCRYPTION/glue_crawler_security_configuration_is_encrypted_check.py new file mode 100644 index 00000000..652c92b4 --- /dev/null +++ b/python/GLUE_CRAWLER_SECURITY_CONFIGURATION_ENCRYPTION/glue_crawler_security_configuration_is_encrypted_check.py @@ -0,0 +1,404 @@ +import boto3, sys, datetime, botocore, json, os +try: + import liblogging +except ImportError: + pass + +############## +# Parameters # +############## + +# Define the default resource to report to Config Rules +DEFAULT_RESOURCE_TYPE = 'AWS::Glue::Crawler' + +# Set to True to get the lambda to assume the Role attached on the Config Service (useful for cross-account). +ASSUME_ROLE_MODE = False + +# Other parameters (no change needed) +CONFIG_ROLE_TIMEOUT_SECONDS = 900 + + + +############# +# Main Code # +############# +def get_all_glue_crawlers(glue_client): + all_crawlers = [] + + crawlers = glue_client.get_crawlers() + all_crawlers += crawlers['Crawlers'] + + while True: + if 'NextToken' in crawlers: + crawlers = glue_client.get_crawlers(NextToken=crawlers['NextToken']) + all_crawlers += crawlers['Crawlers'] + else: + break + + return all_crawlers + +def evaluate_compliance(event, configuration_item, valid_rule_parameters): + glue_client = boto3.client('glue') + evaluations = [] + + all_crawlers = get_all_glue_crawlers(glue_client) + + # No log group exists + if not all_crawlers: + return None + + for crawler in all_crawlers: + try: + configuration_name = crawler['CrawlerSecurityConfiguration'] + + #get encryption status dictionary + encryption_status = check_security_configuration(configuration_name, glue_client) + #check s3 and cloudwatch encryption for security configuration + s3_encryption_config = encryption_status['S3Encryption'] + cloudwatch_encryption_config = encryption_status['CloudWatchEncryption'] + + #if encryption is not enabled for both then put into non-compliant evaluation + if s3_encryption_config[next(iter(s3_encryption_config))] == 'DISABLED' or cloudwatch_encryption_config[next(iter(cloudwatch_encryption_config))] == 'DISABLED': + evaluations.append(build_evaluation(crawler['Name'], + 'NON_COMPLIANT', + event, + annotation="This crawler's security configuration is not encrypted for S3 and CloudWatch")) + + else: + evaluations.append(build_evaluation(crawler['Name'], + 'COMPLIANT', + event, + annotation="This crawler's security configuration is encrypted for S3 and CloudWatch")) + + except KeyError: #security configuration doesn't exist + evaluations.append(build_evaluation(crawler['Name'], + 'NON_COMPLIANT', + event, + annotation="This crawler's doesnt have a security configuration")) + + + print(evaluations[0]) + return evaluations + +def check_security_configuration(security_configuration_name, glue_client): + """This function checks for S3 and Cloudwatch Encryption on the given security configuration and + returns a dictionary of the values""" + + response = glue_client.get_security_configuration( + Name=security_configuration_name + ) + + encryption_status = {} + encryption_config = response['SecurityConfiguration']['EncryptionConfiguration'] + + s3_encryption_config = encryption_config['S3Encryption'][0] + encryption_status['S3Encryption'] = s3_encryption_config + + cloudwatch_encryption_config = encryption_config['CloudWatchEncryption'] + encryption_status['CloudWatchEncryption'] = cloudwatch_encryption_config + + return encryption_status + +def evaluate_parameters(rule_parameters): + if 'KmsKeyId' not in rule_parameters: + return {} + + if 'arn:aws:kms' not in rule_parameters['KmsKeyId']: + raise ValueError('Invalid value for paramter KmsKeyId, Expected KMS Key ARN') + + return rule_parameters + +#################### +# Helper Functions # +#################### + +# Build an error to be displayed in the logs when the parameter is invalid. +def build_parameters_value_error_response(ex): + """Return an error dictionary when the evaluate_parameters() raises a ValueError. + Keyword arguments: + ex -- Exception text + """ + return build_error_response(internal_error_message="Parameter value is invalid", + internal_error_details="An ValueError was raised during the validation of the Parameter value", + customer_error_code="InvalidParameterValueException", + customer_error_message=str(ex)) + +# This gets the client after assuming the Config service role +# either in the same AWS account or cross-account. +def get_client(service, event): + """Return the service boto client. It should be used instead of directly calling the client. + Keyword arguments: + service -- the service name used for calling the boto.client() + event -- the event variable given in the lambda handler + """ + if not ASSUME_ROLE_MODE: + return boto3.client(service) + credentials = get_assume_role_credentials(event["executionRoleArn"]) + return boto3.client(service, aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'] + ) + +# This generate an evaluation for config +def build_evaluation(resource_id, compliance_type, event, resource_type=DEFAULT_RESOURCE_TYPE, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on scheduled rules. + Keyword arguments: + resource_id -- the unique id of the resource to report + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + event -- the event variable given in the lambda handler + resource_type -- the CloudFormation resource type (or AWS::::Account) to report on the rule (default DEFAULT_RESOURCE_TYPE) + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_cc = {} + if annotation: + eval_cc['Annotation'] = annotation + eval_cc['ComplianceResourceType'] = resource_type + eval_cc['ComplianceResourceId'] = resource_id + eval_cc['ComplianceType'] = compliance_type + eval_cc['OrderingTimestamp'] = str(json.loads(event['invokingEvent'])['notificationCreationTime']) + return eval_cc + +def build_evaluation_from_config_item(configuration_item, compliance_type, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on configuration change rules. + Keyword arguments: + configuration_item -- the configurationItem dictionary in the invokingEvent + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_ci = {} + if annotation: + eval_ci['Annotation'] = annotation + eval_ci['ComplianceResourceType'] = configuration_item['resourceType'] + eval_ci['ComplianceResourceId'] = configuration_item['resourceId'] + eval_ci['ComplianceType'] = compliance_type + eval_ci['OrderingTimestamp'] = configuration_item['configurationItemCaptureTime'] + return eval_ci + +#################### +# Boilerplate Code # +#################### + +# Helper function used to validate input +def check_defined(reference, reference_name): + if not reference: + raise Exception('Error: ', reference_name, 'is not defined') + return reference + +# Check whether the message is OversizedConfigurationItemChangeNotification or not +def is_oversized_changed_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'OversizedConfigurationItemChangeNotification' + +# Check whether the message is a ScheduledNotification or not. +def is_scheduled_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'ScheduledNotification' + +# Get configurationItem using getResourceConfigHistory API +# in case of OversizedConfigurationItemChangeNotification +def get_configuration(resource_type, resource_id, configuration_capture_time): + result = AWS_CONFIG_CLIENT.get_resource_config_history( + resourceType=resource_type, + resourceId=resource_id, + laterTime=configuration_capture_time, + limit=1) + configuration_item = result['configurationItems'][0] + return convert_api_configuration(configuration_item) + +# Convert from the API model to the original invocation model +def convert_api_configuration(configuration_item): + for k, v in configuration_item.items(): + if isinstance(v, datetime.datetime): + configuration_item[k] = str(v) + configuration_item['awsAccountId'] = configuration_item['accountId'] + configuration_item['ARN'] = configuration_item['arn'] + configuration_item['configurationStateMd5Hash'] = configuration_item['configurationItemMD5Hash'] + configuration_item['configurationItemVersion'] = configuration_item['version'] + configuration_item['configuration'] = json.loads(configuration_item['configuration']) + if 'relationships' in configuration_item: + for i in range(len(configuration_item['relationships'])): + configuration_item['relationships'][i]['name'] = configuration_item['relationships'][i]['relationshipName'] + return configuration_item + +# Based on the type of message get the configuration item +# either from configurationItem in the invoking event +# or using the getResourceConfigHistiry API in getConfiguration function. +def get_configuration_item(invoking_event): + check_defined(invoking_event, 'invokingEvent') + if is_oversized_changed_notification(invoking_event['messageType']): + configuration_item_summary = check_defined(invoking_event['configuration_item_summary'], 'configurationItemSummary') + return get_configuration(configuration_item_summary['resourceType'], configuration_item_summary['resourceId'], configuration_item_summary['configurationItemCaptureTime']) + if is_scheduled_notification(invoking_event['messageType']): + return None + return check_defined(invoking_event['configurationItem'], 'configurationItem') + +# Check whether the resource has been deleted. If it has, then the evaluation is unnecessary. +def is_applicable(configuration_item, event): + try: + check_defined(configuration_item, 'configurationItem') + check_defined(event, 'event') + except: + return True + status = configuration_item['configurationItemStatus'] + event_left_scope = event['eventLeftScope'] + if status == 'ResourceDeleted': + print("Resource Deleted, setting Compliance Status to NOT_APPLICABLE.") + return status in ('OK', 'ResourceDiscovered') and not event_left_scope + +def get_assume_role_credentials(role_arn): + sts_client = boto3.client('sts') + try: + assume_role_response = sts_client.assume_role(RoleArn=role_arn, + RoleSessionName="configLambdaExecution", + DurationSeconds=CONFIG_ROLE_TIMEOUT_SECONDS) + if 'liblogging' in sys.modules: + liblogging.logSession(role_arn, assume_role_response) + return assume_role_response['Credentials'] + except botocore.exceptions.ClientError as ex: + # Scrub error message for any internal account info leaks + print(str(ex)) + if 'AccessDenied' in ex.response['Error']['Code']: + ex.response['Error']['Message'] = "AWS Config does not have permission to assume the IAM role." + else: + ex.response['Error']['Message'] = "InternalError" + ex.response['Error']['Code'] = "InternalError" + raise ex + +# This removes older evaluation (usually useful for periodic rule not reporting on AWS::::Account). +def clean_up_old_evaluations(latest_evaluations, event): + + cleaned_evaluations = [] + + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100) + + old_eval_list = [] + + while True: + for old_result in old_eval['EvaluationResults']: + old_eval_list.append(old_result) + if 'NextToken' in old_eval: + next_token = old_eval['NextToken'] + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100, + NextToken=next_token) + else: + break + + for old_eval in old_eval_list: + old_resource_id = old_eval['EvaluationResultIdentifier']['EvaluationResultQualifier']['ResourceId'] + newer_founded = False + for latest_eval in latest_evaluations: + if old_resource_id == latest_eval['ComplianceResourceId']: + newer_founded = True + if not newer_founded: + cleaned_evaluations.append(build_evaluation(old_resource_id, "NOT_APPLICABLE", event)) + + return cleaned_evaluations + latest_evaluations + +def lambda_handler(event, context): + if 'liblogging' in sys.modules: + liblogging.logEvent(event) + + global AWS_CONFIG_CLIENT + + #print(event) + check_defined(event, 'event') + invoking_event = json.loads(event['invokingEvent']) + rule_parameters = {} + if 'ruleParameters' in event: + rule_parameters = json.loads(event['ruleParameters']) + + try: + valid_rule_parameters = evaluate_parameters(rule_parameters) + except ValueError as ex: + return build_parameters_value_error_response(ex) + + try: + AWS_CONFIG_CLIENT = get_client('config', event) + if invoking_event['messageType'] in ['ConfigurationItemChangeNotification', 'ScheduledNotification', 'OversizedConfigurationItemChangeNotification']: + configuration_item = get_configuration_item(invoking_event) + if is_applicable(configuration_item, event): + compliance_result = evaluate_compliance(event, configuration_item, valid_rule_parameters) + else: + compliance_result = "NOT_APPLICABLE" + else: + return build_internal_error_response('Unexpected message type', str(invoking_event)) + except botocore.exceptions.ClientError as ex: + if is_internal_error(ex): + return build_internal_error_response("Unexpected error while completing API request", str(ex)) + return build_error_response("Customer error while making API request", str(ex), ex.response['Error']['Code'], ex.response['Error']['Message']) + except ValueError as ex: + return build_internal_error_response(str(ex), str(ex)) + + evaluations = [] + latest_evaluations = [] + + if not compliance_result: + latest_evaluations.append(build_evaluation(event['accountId'], "NOT_APPLICABLE", event, resource_type='AWS::::Account')) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, str): + if configuration_item: + evaluations.append(build_evaluation_from_config_item(configuration_item, compliance_result)) + else: + evaluations.append(build_evaluation(event['accountId'], compliance_result, event, resource_type=DEFAULT_RESOURCE_TYPE)) + elif isinstance(compliance_result, list): + for evaluation in compliance_result: + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in evaluation: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + + if not missing_fields: + latest_evaluations.append(evaluation) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, dict): + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in compliance_result: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + if not missing_fields: + evaluations.append(compliance_result) + else: + evaluations.append(build_evaluation_from_config_item(configuration_item, 'NOT_APPLICABLE')) + + # Put together the request that reports the evaluation status + result_token = event['resultToken'] + test_mode = False + if result_token == 'TESTMODE': + # Used solely for RDK test to skip actual put_evaluation API call + test_mode = True + + # Invoke the Config API to report the result of the evaluation + evaluation_copy = [] + evaluation_copy = evaluations[:] + while evaluation_copy: + AWS_CONFIG_CLIENT.put_evaluations(Evaluations=evaluation_copy[:100], ResultToken=result_token, TestMode=test_mode) + del evaluation_copy[:100] + + # Used solely for RDK test to be able to test Lambda function + return evaluations + +def is_internal_error(exception): + return ((not isinstance(exception, botocore.exceptions.ClientError)) or exception.response['Error']['Code'].startswith('5') + or 'InternalError' in exception.response['Error']['Code'] or 'ServiceError' in exception.response['Error']['Code']) + +def build_internal_error_response(internal_error_message, internal_error_details=None): + return build_error_response(internal_error_message, internal_error_details, 'InternalError', 'InternalError') + +def build_error_response(internal_error_message, internal_error_details=None, customer_error_code=None, customer_error_message=None): + error_response = { + 'internalErrorMessage': internal_error_message, + 'internalErrorDetails': internal_error_details, + 'customerErrorMessage': customer_error_message, + 'customerErrorCode': customer_error_code + } + print(error_response) + return error_response + diff --git a/python/GLUE_CRAWLER_SECURITY_CONFIGURATION_ENCRYPTION/remediate_crawler_security_config_encryption.py b/python/GLUE_CRAWLER_SECURITY_CONFIGURATION_ENCRYPTION/remediate_crawler_security_config_encryption.py new file mode 100644 index 00000000..ba4674ce --- /dev/null +++ b/python/GLUE_CRAWLER_SECURITY_CONFIGURATION_ENCRYPTION/remediate_crawler_security_config_encryption.py @@ -0,0 +1,123 @@ +import boto3, logging, sys +from botocore.exceptions import ClientError + +def lambda_handler(event, context): + print(event) + + glue_crawler_name = event['ResourceId'] + glue_client = boto3.client('glue') + + configuration_name = get_crawler_security_configuartion(glue_crawler_name, glue_client) + if not configuration_name: + logging.info('Glue crawler has no security configuration, creating a new one now') + new_security_configuration = create_security_configuration(glue_client, glue_crawler_name) + change_crawler_security_configuration(glue_client, glue_crawler_name, new_security_configuration) + + else: #there is an existing configuration + #get encryption status dictionary + encryption_status = check_security_configuration(configuration_name, glue_client) + + #check s3 and cloudwatch encryption for security configuration + s3_encryption_config = encryption_status['S3Encryption'] + cloudwatch_encryption_config = encryption_status['CloudWatchEncryption'] + + #if encryption is not enabled for both, then create a new security configuration and update the glue job + if s3_encryption_config[next(iter(s3_encryption_config))] == 'DISABLED' or cloudwatch_encryption_config[next(iter(cloudwatch_encryption_config))] == 'DISABLED': + new_security_configuration = create_security_configuration(glue_client, glue_crawler_name) + change_crawler_security_configuration(glue_client, glue_crawler_name, new_security_configuration) + logging.info(glue_crawler_name + "'s security configuration has been updated to a new one called " + new_security_configuration) + + else: logging.info('S3 and CloudWatch Encryption are both Enabled for ' + str(configuration_name)) + +def change_crawler_security_configuration(glue_client, glue_crawler_name, new_security_configuration_name): + """This function changes the security configuartion of the glue crawler to the new_security_configuration_name""" + glue_client.update_crawler( + Name=glue_crawler_name, + CrawlerSecurityConfiguration=new_security_configuration_name + ) + + +def check_security_configuration(security_configuration_name, glue_client): + """This function checks for S3 and Cloudwatch Encryption on the given security configuration and + returns a dictionary of the values""" + + response = glue_client.get_security_configuration( + Name=security_configuration_name + ) + + encryption_status = {} + encryption_config = response['SecurityConfiguration']['EncryptionConfiguration'] + + s3_encryption_config = encryption_config['S3Encryption'][0] + encryption_status['S3Encryption'] = s3_encryption_config + + cloudwatch_encryption_config = encryption_config['CloudWatchEncryption'] + encryption_status['CloudWatchEncryption'] = cloudwatch_encryption_config + + return encryption_status + +def get_crawler_security_configuartion(glue_crawler_name, glue_client): + """This function returns the security configuration name of the given glue job""" + response = glue_client.get_crawler( + Name = glue_crawler_name + ) + try: + security_configuration = response['Crawler']['CrawlerSecurityConfiguration'] + except: + security_configuration = {} + + return security_configuration + +def create_security_configuration(glue_client, glue_crawler_name): + """This function creates a new encrypted security configuration""" + + kms_key_arn = create_kms_key() + security_config_name = glue_crawler_name + '_SecurityConfiguration' + try: + response = glue_client.create_security_configuration( + Name= security_config_name, + EncryptionConfiguration={ + 'S3Encryption': [ + { + 'S3EncryptionMode': 'SSE-KMS', + 'KmsKeyArn': kms_key_arn + }, + ], + 'CloudWatchEncryption': { + 'CloudWatchEncryptionMode': 'SSE-KMS', + 'KmsKeyArn': kms_key_arn + } + }, + ) + return response['Name'] + + except ClientError as e: + if e.response['Error']['Code'] == 'AlreadyExistsException': + return security_config_name + else: + logging.critical('Security Configuration Creation Failed') + sys.exit(1) + + +def create_kms_key(): + """This function creates a symmetric KMS key and returns its ARN""" + kms_client = boto3.client('kms') + response = kms_client.create_key( + # Policy='string', + Description='This key was created to secure and encrypt AWS Glue', + KeyUsage='ENCRYPT_DECRYPT', #Alternative is 'SIGN_VERIFY'| + CustomerMasterKeySpec= 'SYMMETRIC_DEFAULT', #Alternatives are 'RSA_2048'|'RSA_3072'|'RSA_4096'|'ECC_NIST_P256'|'ECC_NIST_P384'|'ECC_NIST_P521'|'ECC_SECG_P256K1'|' + Origin='AWS_KMS', #Alternatives are |'EXTERNAL'|'AWS_CLOUDHSM', + # CustomKeyStoreId='string', + # BypassPolicyLockoutSafetyCheck=True|False, + Tags=[ + { + 'TagKey': 'string', + 'TagValue': 'string' + }, + ] + ) + + kms_key_arn = response['KeyMetadata']['Arn'] + logging.info('KMS KEY CREATED: ' + str(kms_key_arn)) + return kms_key_arn \ No newline at end of file diff --git a/python/GLUE_DATA_CATALOG_ENCRYPTION/glue_data_catalog_is_encrypted_check.py b/python/GLUE_DATA_CATALOG_ENCRYPTION/glue_data_catalog_is_encrypted_check.py new file mode 100644 index 00000000..d78143fc --- /dev/null +++ b/python/GLUE_DATA_CATALOG_ENCRYPTION/glue_data_catalog_is_encrypted_check.py @@ -0,0 +1,344 @@ +import boto3, sys, datetime, botocore, json, os + +############## +# Parameters # +############## + +# Define the default resource to report to Config Rules +DEFAULT_RESOURCE_TYPE = 'AWS::Glue::DataCatalogEncryptionSettings' + +# Set to True to get the lambda to assume the Role attached on the Config Service (useful for cross-account). +ASSUME_ROLE_MODE = False + +# Other parameters (no change needed) +CONFIG_ROLE_TIMEOUT_SECONDS = 900 + +############# +# Main Code # +############# +def evaluate_compliance(event, configuration_item, valid_rule_parameters): + glue_client = boto3.client('glue') + evaluations = [] + + response = glue_client.get_data_catalog_encryption_settings() + catalog_encryption = response['DataCatalogEncryptionSettings']['EncryptionAtRest']['CatalogEncryptionMode'] + password_encryption = response['DataCatalogEncryptionSettings']['ConnectionPasswordEncryption']['ReturnConnectionPasswordEncrypted'] + + #if encryption is not enabled for both then put into non-compliant evaluation + if catalog_encryption == 'DISABLED' or password_encryption == False: + evaluations.append(build_evaluation('Data Catalog Settings', + 'NON_COMPLIANT', + event, + annotation="The Glue Data Catalog is not encrypted")) + + else: + evaluations.append(build_evaluation('Data Catalog Settings', + 'COMPLIANT', + event, + annotation="The Glue Data Catalog is not encrypted")) + + return evaluations + + +def evaluate_parameters(rule_parameters): + if 'KmsKeyId' not in rule_parameters: + return {} + + if 'arn:aws:kms' not in rule_parameters['KmsKeyId']: + raise ValueError('Invalid value for paramter KmsKeyId, Expected KMS Key ARN') + + return rule_parameters + +#################### +# Helper Functions # +#################### + +# Build an error to be displayed in the logs when the parameter is invalid. +def build_parameters_value_error_response(ex): + """Return an error dictionary when the evaluate_parameters() raises a ValueError. + Keyword arguments: + ex -- Exception text + """ + return build_error_response(internal_error_message="Parameter value is invalid", + internal_error_details="An ValueError was raised during the validation of the Parameter value", + customer_error_code="InvalidParameterValueException", + customer_error_message=str(ex)) + +# This gets the client after assuming the Config service role +# either in the same AWS account or cross-account. +def get_client(service, event): + """Return the service boto client. It should be used instead of directly calling the client. + Keyword arguments: + service -- the service name used for calling the boto.client() + event -- the event variable given in the lambda handler + """ + if not ASSUME_ROLE_MODE: + return boto3.client(service) + credentials = get_assume_role_credentials(event["executionRoleArn"]) + return boto3.client(service, aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'] + ) + +# This generate an evaluation for config +def build_evaluation(resource_id, compliance_type, event, resource_type=DEFAULT_RESOURCE_TYPE, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on scheduled rules. + Keyword arguments: + resource_id -- the unique id of the resource to report + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + event -- the event variable given in the lambda handler + resource_type -- the CloudFormation resource type (or AWS::::Account) to report on the rule (default DEFAULT_RESOURCE_TYPE) + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_cc = {} + if annotation: + eval_cc['Annotation'] = annotation + eval_cc['ComplianceResourceType'] = resource_type + eval_cc['ComplianceResourceId'] = resource_id + eval_cc['ComplianceType'] = compliance_type + eval_cc['OrderingTimestamp'] = str(json.loads(event['invokingEvent'])['notificationCreationTime']) + return eval_cc + +def build_evaluation_from_config_item(configuration_item, compliance_type, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on configuration change rules. + Keyword arguments: + configuration_item -- the configurationItem dictionary in the invokingEvent + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_ci = {} + if annotation: + eval_ci['Annotation'] = annotation + eval_ci['ComplianceResourceType'] = configuration_item['resourceType'] + eval_ci['ComplianceResourceId'] = configuration_item['resourceId'] + eval_ci['ComplianceType'] = compliance_type + eval_ci['OrderingTimestamp'] = configuration_item['configurationItemCaptureTime'] + return eval_ci + +#################### +# Boilerplate Code # +#################### + +# Helper function used to validate input +def check_defined(reference, reference_name): + if not reference: + raise Exception('Error: ', reference_name, 'is not defined') + return reference + +# Check whether the message is OversizedConfigurationItemChangeNotification or not +def is_oversized_changed_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'OversizedConfigurationItemChangeNotification' + +# Check whether the message is a ScheduledNotification or not. +def is_scheduled_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'ScheduledNotification' + +# Get configurationItem using getResourceConfigHistory API +# in case of OversizedConfigurationItemChangeNotification +def get_configuration(resource_type, resource_id, configuration_capture_time): + result = AWS_CONFIG_CLIENT.get_resource_config_history( + resourceType=resource_type, + resourceId=resource_id, + laterTime=configuration_capture_time, + limit=1) + configuration_item = result['configurationItems'][0] + return convert_api_configuration(configuration_item) + +# Convert from the API model to the original invocation model +def convert_api_configuration(configuration_item): + for k, v in configuration_item.items(): + if isinstance(v, datetime.datetime): + configuration_item[k] = str(v) + configuration_item['awsAccountId'] = configuration_item['accountId'] + configuration_item['ARN'] = configuration_item['arn'] + configuration_item['configurationStateMd5Hash'] = configuration_item['configurationItemMD5Hash'] + configuration_item['configurationItemVersion'] = configuration_item['version'] + configuration_item['configuration'] = json.loads(configuration_item['configuration']) + if 'relationships' in configuration_item: + for i in range(len(configuration_item['relationships'])): + configuration_item['relationships'][i]['name'] = configuration_item['relationships'][i]['relationshipName'] + return configuration_item + +# Based on the type of message get the configuration item +# either from configurationItem in the invoking event +# or using the getResourceConfigHistiry API in getConfiguration function. +def get_configuration_item(invoking_event): + check_defined(invoking_event, 'invokingEvent') + if is_oversized_changed_notification(invoking_event['messageType']): + configuration_item_summary = check_defined(invoking_event['configuration_item_summary'], 'configurationItemSummary') + return get_configuration(configuration_item_summary['resourceType'], configuration_item_summary['resourceId'], configuration_item_summary['configurationItemCaptureTime']) + if is_scheduled_notification(invoking_event['messageType']): + return None + return check_defined(invoking_event['configurationItem'], 'configurationItem') + +# Check whether the resource has been deleted. If it has, then the evaluation is unnecessary. +def is_applicable(configuration_item, event): + try: + check_defined(configuration_item, 'configurationItem') + check_defined(event, 'event') + except: + return True + status = configuration_item['configurationItemStatus'] + event_left_scope = event['eventLeftScope'] + if status == 'ResourceDeleted': + print("Resource Deleted, setting Compliance Status to NOT_APPLICABLE.") + return status in ('OK', 'ResourceDiscovered') and not event_left_scope + +def get_assume_role_credentials(role_arn): + sts_client = boto3.client('sts') + try: + assume_role_response = sts_client.assume_role(RoleArn=role_arn, + RoleSessionName="configLambdaExecution", + DurationSeconds=CONFIG_ROLE_TIMEOUT_SECONDS) + if 'liblogging' in sys.modules: + liblogging.logSession(role_arn, assume_role_response) + return assume_role_response['Credentials'] + except botocore.exceptions.ClientError as ex: + # Scrub error message for any internal account info leaks + print(str(ex)) + if 'AccessDenied' in ex.response['Error']['Code']: + ex.response['Error']['Message'] = "AWS Config does not have permission to assume the IAM role." + else: + ex.response['Error']['Message'] = "InternalError" + ex.response['Error']['Code'] = "InternalError" + raise ex + +# This removes older evaluation (usually useful for periodic rule not reporting on AWS::::Account). +def clean_up_old_evaluations(latest_evaluations, event): + + cleaned_evaluations = [] + + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100) + + old_eval_list = [] + + while True: + for old_result in old_eval['EvaluationResults']: + old_eval_list.append(old_result) + if 'NextToken' in old_eval: + next_token = old_eval['NextToken'] + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100, + NextToken=next_token) + else: + break + + for old_eval in old_eval_list: + old_resource_id = old_eval['EvaluationResultIdentifier']['EvaluationResultQualifier']['ResourceId'] + newer_founded = False + for latest_eval in latest_evaluations: + if old_resource_id == latest_eval['ComplianceResourceId']: + newer_founded = True + if not newer_founded: + cleaned_evaluations.append(build_evaluation(old_resource_id, "NOT_APPLICABLE", event)) + + return cleaned_evaluations + latest_evaluations + +def lambda_handler(event, context): + if 'liblogging' in sys.modules: + liblogging.logEvent(event) + + global AWS_CONFIG_CLIENT + + #print(event) + check_defined(event, 'event') + invoking_event = json.loads(event['invokingEvent']) + rule_parameters = {} + if 'ruleParameters' in event: + rule_parameters = json.loads(event['ruleParameters']) + + try: + valid_rule_parameters = evaluate_parameters(rule_parameters) + except ValueError as ex: + return build_parameters_value_error_response(ex) + + try: + AWS_CONFIG_CLIENT = get_client('config', event) + if invoking_event['messageType'] in ['ConfigurationItemChangeNotification', 'ScheduledNotification', 'OversizedConfigurationItemChangeNotification']: + configuration_item = get_configuration_item(invoking_event) + if is_applicable(configuration_item, event): + compliance_result = evaluate_compliance(event, configuration_item, valid_rule_parameters) + else: + compliance_result = "NOT_APPLICABLE" + else: + return build_internal_error_response('Unexpected message type', str(invoking_event)) + except botocore.exceptions.ClientError as ex: + if is_internal_error(ex): + return build_internal_error_response("Unexpected error while completing API request", str(ex)) + return build_error_response("Customer error while making API request", str(ex), ex.response['Error']['Code'], ex.response['Error']['Message']) + except ValueError as ex: + return build_internal_error_response(str(ex), str(ex)) + + evaluations = [] + latest_evaluations = [] + + if not compliance_result: + latest_evaluations.append(build_evaluation(event['accountId'], "NOT_APPLICABLE", event, resource_type='AWS::::Account')) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, str): + if configuration_item: + evaluations.append(build_evaluation_from_config_item(configuration_item, compliance_result)) + else: + evaluations.append(build_evaluation(event['accountId'], compliance_result, event, resource_type=DEFAULT_RESOURCE_TYPE)) + elif isinstance(compliance_result, list): + for evaluation in compliance_result: + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in evaluation: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + + if not missing_fields: + latest_evaluations.append(evaluation) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, dict): + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in compliance_result: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + if not missing_fields: + evaluations.append(compliance_result) + else: + evaluations.append(build_evaluation_from_config_item(configuration_item, 'NOT_APPLICABLE')) + + # Put together the request that reports the evaluation status + result_token = event['resultToken'] + test_mode = False + if result_token == 'TESTMODE': + # Used solely for RDK test to skip actual put_evaluation API call + test_mode = True + + # Invoke the Config API to report the result of the evaluation + evaluation_copy = [] + evaluation_copy = evaluations[:] + while evaluation_copy: + AWS_CONFIG_CLIENT.put_evaluations(Evaluations=evaluation_copy[:100], ResultToken=result_token, TestMode=test_mode) + del evaluation_copy[:100] + + # Used solely for RDK test to be able to test Lambda function + return evaluations + +def is_internal_error(exception): + return ((not isinstance(exception, botocore.exceptions.ClientError)) or exception.response['Error']['Code'].startswith('5') + or 'InternalError' in exception.response['Error']['Code'] or 'ServiceError' in exception.response['Error']['Code']) + +def build_internal_error_response(internal_error_message, internal_error_details=None): + return build_error_response(internal_error_message, internal_error_details, 'InternalError', 'InternalError') + +def build_error_response(internal_error_message, internal_error_details=None, customer_error_code=None, customer_error_message=None): + error_response = { + 'internalErrorMessage': internal_error_message, + 'internalErrorDetails': internal_error_details, + 'customerErrorMessage': customer_error_message, + 'customerErrorCode': customer_error_code + } + print(error_response) + return error_response diff --git a/python/GLUE_DATA_CATALOG_ENCRYPTION/remediate_data_catalog_encryption.py b/python/GLUE_DATA_CATALOG_ENCRYPTION/remediate_data_catalog_encryption.py new file mode 100644 index 00000000..4f42f192 --- /dev/null +++ b/python/GLUE_DATA_CATALOG_ENCRYPTION/remediate_data_catalog_encryption.py @@ -0,0 +1,38 @@ +import boto3, logging + +def lambda_handler(event, context): + glue_client = boto3.client('glue') + kms_key_arn = create_kms_key() #create kmskey + + glue_client.put_data_catalog_encryption_settings( + DataCatalogEncryptionSettings={ + 'EncryptionAtRest': { + 'CatalogEncryptionMode': 'SSE-KMS', + 'SseAwsKmsKeyId': kms_key_arn + }, + 'ConnectionPasswordEncryption': { + 'ReturnConnectionPasswordEncrypted': True, + 'AwsKmsKeyId': kms_key_arn + } + } +) + +def create_kms_key(): + """This function creates a symmetric KMS key and returns its ARN""" + kms_client = boto3.client('kms') + response = kms_client.create_key( + Description='This key was created to secure and encrypt AWS Glue data catalog', + KeyUsage='ENCRYPT_DECRYPT', #Alternative is 'SIGN_VERIFY'| + CustomerMasterKeySpec= 'SYMMETRIC_DEFAULT', #Alternatives are 'RSA_2048'|'RSA_3072'|'RSA_4096'|'ECC_NIST_P256'|'ECC_NIST_P384'|'ECC_NIST_P521'|'ECC_SECG_P256K1'|' + Origin='AWS_KMS', #Alternatives are |'EXTERNAL'|'AWS_CLOUDHSM', + Tags=[ + { + 'TagKey': 'string', + 'TagValue': 'string' + }, + ] + ) + + kms_key_arn = response['KeyMetadata']['Arn'] + logging.info('KMS KEY CREATED: ' + str(kms_key_arn)) + return kms_key_arn \ No newline at end of file diff --git a/python/GLUE_JOB_SECURITY_CONFIGURATION_ENCRYPTION/glue_job_security_configuration_is_encrypted_check.py b/python/GLUE_JOB_SECURITY_CONFIGURATION_ENCRYPTION/glue_job_security_configuration_is_encrypted_check.py new file mode 100644 index 00000000..cc148ec1 --- /dev/null +++ b/python/GLUE_JOB_SECURITY_CONFIGURATION_ENCRYPTION/glue_job_security_configuration_is_encrypted_check.py @@ -0,0 +1,400 @@ +import boto3, sys, datetime, botocore, json, os + +############## +# Parameters # +############## + +# Define the default resource to report to Config Rules +DEFAULT_RESOURCE_TYPE = 'AWS::Glue::Job' + +# Set to True to get the lambda to assume the Role attached on the Config Service (useful for cross-account). +ASSUME_ROLE_MODE = False + +# Other parameters (no change needed) +CONFIG_ROLE_TIMEOUT_SECONDS = 900 + +############# +# Main Code # +############# +def get_all_glue_jobs(glue_client): + all_jobs = [] + + jobs = glue_client.get_jobs() + all_jobs += jobs['Jobs'] + + while True: + if 'NextToken' in jobs: + jobs = glue_client.get_jobs(NextToken=jobs['NextToken']) + all_jobs += jobs['Jobs'] + else: + break + + return all_jobs + +def evaluate_compliance(event, configuration_item, valid_rule_parameters): + glue_client = boto3.client('glue') + evaluations = [] + + all_jobs = get_all_glue_jobs(glue_client) + + # No log group exists + if not all_jobs: + return None + + for job in all_jobs: + print(job) + try: + configuration_name = job['SecurityConfiguration'] + + #get encryption status dictionary + encryption_status = check_security_configuration(configuration_name, glue_client) + #check s3 and cloudwatch encryption for security configuration + s3_encryption_config = encryption_status['S3Encryption'] + cloudwatch_encryption_config = encryption_status['CloudWatchEncryption'] + + #if encryption is not enabled for both then put into non-compliant evaluation + if s3_encryption_config[next(iter(s3_encryption_config))] == 'DISABLED' or cloudwatch_encryption_config[next(iter(cloudwatch_encryption_config))] == 'DISABLED': + evaluations.append(build_evaluation(job['Name'], + 'NON_COMPLIANT', + event, + annotation="This job's security configuration is not encrypted for S3 and CloudWatch")) + + else: + evaluations.append(build_evaluation(job['Name'], + 'COMPLIANT', + event, + annotation="This job's security configuration is encrypted for S3 and CloudWatch")) + + except KeyError: #security configuration doesn't exist + evaluations.append(build_evaluation(job['Name'], + 'NON_COMPLIANT', + event, + annotation="This job's doesnt have a security configuration")) + + + print(evaluations[0]) + return evaluations + + +def check_security_configuration(security_configuration_name, glue_client): + """This function checks for S3 and Cloudwatch Encryption on the given security configuration and + returns a dictionary of the values""" + + response = glue_client.get_security_configuration( + Name=security_configuration_name + ) + + encryption_status = {} + encryption_config = response['SecurityConfiguration']['EncryptionConfiguration'] + + s3_encryption_config = encryption_config['S3Encryption'][0] + encryption_status['S3Encryption'] = s3_encryption_config + + cloudwatch_encryption_config = encryption_config['CloudWatchEncryption'] + encryption_status['CloudWatchEncryption'] = cloudwatch_encryption_config + + return encryption_status + + +def evaluate_parameters(rule_parameters): + if 'KmsKeyId' not in rule_parameters: + return {} + + if 'arn:aws:kms' not in rule_parameters['KmsKeyId']: + raise ValueError('Invalid value for paramter KmsKeyId, Expected KMS Key ARN') + + return rule_parameters + +#################### +# Helper Functions # +#################### + +# Build an error to be displayed in the logs when the parameter is invalid. +def build_parameters_value_error_response(ex): + """Return an error dictionary when the evaluate_parameters() raises a ValueError. + Keyword arguments: + ex -- Exception text + """ + return build_error_response(internal_error_message="Parameter value is invalid", + internal_error_details="An ValueError was raised during the validation of the Parameter value", + customer_error_code="InvalidParameterValueException", + customer_error_message=str(ex)) + +# This gets the client after assuming the Config service role +# either in the same AWS account or cross-account. +def get_client(service, event): + """Return the service boto client. It should be used instead of directly calling the client. + Keyword arguments: + service -- the service name used for calling the boto.client() + event -- the event variable given in the lambda handler + """ + if not ASSUME_ROLE_MODE: + return boto3.client(service) + credentials = get_assume_role_credentials(event["executionRoleArn"]) + return boto3.client(service, aws_access_key_id=credentials['AccessKeyId'], + aws_secret_access_key=credentials['SecretAccessKey'], + aws_session_token=credentials['SessionToken'] + ) + +# This generate an evaluation for config +def build_evaluation(resource_id, compliance_type, event, resource_type=DEFAULT_RESOURCE_TYPE, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on scheduled rules. + Keyword arguments: + resource_id -- the unique id of the resource to report + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + event -- the event variable given in the lambda handler + resource_type -- the CloudFormation resource type (or AWS::::Account) to report on the rule (default DEFAULT_RESOURCE_TYPE) + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_cc = {} + if annotation: + eval_cc['Annotation'] = annotation + eval_cc['ComplianceResourceType'] = resource_type + eval_cc['ComplianceResourceId'] = resource_id + eval_cc['ComplianceType'] = compliance_type + eval_cc['OrderingTimestamp'] = str(json.loads(event['invokingEvent'])['notificationCreationTime']) + return eval_cc + +def build_evaluation_from_config_item(configuration_item, compliance_type, annotation=None): + """Form an evaluation as a dictionary. Usually suited to report on configuration change rules. + Keyword arguments: + configuration_item -- the configurationItem dictionary in the invokingEvent + compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE + annotation -- an annotation to be added to the evaluation (default None) + """ + eval_ci = {} + if annotation: + eval_ci['Annotation'] = annotation + eval_ci['ComplianceResourceType'] = configuration_item['resourceType'] + eval_ci['ComplianceResourceId'] = configuration_item['resourceId'] + eval_ci['ComplianceType'] = compliance_type + eval_ci['OrderingTimestamp'] = configuration_item['configurationItemCaptureTime'] + return eval_ci + +#################### +# Boilerplate Code # +#################### + +# Helper function used to validate input +def check_defined(reference, reference_name): + if not reference: + raise Exception('Error: ', reference_name, 'is not defined') + return reference + +# Check whether the message is OversizedConfigurationItemChangeNotification or not +def is_oversized_changed_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'OversizedConfigurationItemChangeNotification' + +# Check whether the message is a ScheduledNotification or not. +def is_scheduled_notification(message_type): + check_defined(message_type, 'messageType') + return message_type == 'ScheduledNotification' + +# Get configurationItem using getResourceConfigHistory API +# in case of OversizedConfigurationItemChangeNotification +def get_configuration(resource_type, resource_id, configuration_capture_time): + result = AWS_CONFIG_CLIENT.get_resource_config_history( + resourceType=resource_type, + resourceId=resource_id, + laterTime=configuration_capture_time, + limit=1) + configuration_item = result['configurationItems'][0] + return convert_api_configuration(configuration_item) + +# Convert from the API model to the original invocation model +def convert_api_configuration(configuration_item): + for k, v in configuration_item.items(): + if isinstance(v, datetime.datetime): + configuration_item[k] = str(v) + configuration_item['awsAccountId'] = configuration_item['accountId'] + configuration_item['ARN'] = configuration_item['arn'] + configuration_item['configurationStateMd5Hash'] = configuration_item['configurationItemMD5Hash'] + configuration_item['configurationItemVersion'] = configuration_item['version'] + configuration_item['configuration'] = json.loads(configuration_item['configuration']) + if 'relationships' in configuration_item: + for i in range(len(configuration_item['relationships'])): + configuration_item['relationships'][i]['name'] = configuration_item['relationships'][i]['relationshipName'] + return configuration_item + +# Based on the type of message get the configuration item +# either from configurationItem in the invoking event +# or using the getResourceConfigHistiry API in getConfiguration function. +def get_configuration_item(invoking_event): + check_defined(invoking_event, 'invokingEvent') + if is_oversized_changed_notification(invoking_event['messageType']): + configuration_item_summary = check_defined(invoking_event['configuration_item_summary'], 'configurationItemSummary') + return get_configuration(configuration_item_summary['resourceType'], configuration_item_summary['resourceId'], configuration_item_summary['configurationItemCaptureTime']) + if is_scheduled_notification(invoking_event['messageType']): + return None + return check_defined(invoking_event['configurationItem'], 'configurationItem') + +# Check whether the resource has been deleted. If it has, then the evaluation is unnecessary. +def is_applicable(configuration_item, event): + try: + check_defined(configuration_item, 'configurationItem') + check_defined(event, 'event') + except: + return True + status = configuration_item['configurationItemStatus'] + event_left_scope = event['eventLeftScope'] + if status == 'ResourceDeleted': + print("Resource Deleted, setting Compliance Status to NOT_APPLICABLE.") + return status in ('OK', 'ResourceDiscovered') and not event_left_scope + +def get_assume_role_credentials(role_arn): + sts_client = boto3.client('sts') + try: + assume_role_response = sts_client.assume_role(RoleArn=role_arn, + RoleSessionName="configLambdaExecution", + DurationSeconds=CONFIG_ROLE_TIMEOUT_SECONDS) + if 'liblogging' in sys.modules: + liblogging.logSession(role_arn, assume_role_response) + return assume_role_response['Credentials'] + except botocore.exceptions.ClientError as ex: + # Scrub error message for any internal account info leaks + print(str(ex)) + if 'AccessDenied' in ex.response['Error']['Code']: + ex.response['Error']['Message'] = "AWS Config does not have permission to assume the IAM role." + else: + ex.response['Error']['Message'] = "InternalError" + ex.response['Error']['Code'] = "InternalError" + raise ex + +# This removes older evaluation (usually useful for periodic rule not reporting on AWS::::Account). +def clean_up_old_evaluations(latest_evaluations, event): + + cleaned_evaluations = [] + + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100) + + old_eval_list = [] + + while True: + for old_result in old_eval['EvaluationResults']: + old_eval_list.append(old_result) + if 'NextToken' in old_eval: + next_token = old_eval['NextToken'] + old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule( + ConfigRuleName=event['configRuleName'], + ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'], + Limit=100, + NextToken=next_token) + else: + break + + for old_eval in old_eval_list: + old_resource_id = old_eval['EvaluationResultIdentifier']['EvaluationResultQualifier']['ResourceId'] + newer_founded = False + for latest_eval in latest_evaluations: + if old_resource_id == latest_eval['ComplianceResourceId']: + newer_founded = True + if not newer_founded: + cleaned_evaluations.append(build_evaluation(old_resource_id, "NOT_APPLICABLE", event)) + + return cleaned_evaluations + latest_evaluations + +def lambda_handler(event, context): + if 'liblogging' in sys.modules: + liblogging.logEvent(event) + + global AWS_CONFIG_CLIENT + + #print(event) + check_defined(event, 'event') + invoking_event = json.loads(event['invokingEvent']) + rule_parameters = {} + if 'ruleParameters' in event: + rule_parameters = json.loads(event['ruleParameters']) + + try: + valid_rule_parameters = evaluate_parameters(rule_parameters) + except ValueError as ex: + return build_parameters_value_error_response(ex) + + try: + AWS_CONFIG_CLIENT = get_client('config', event) + if invoking_event['messageType'] in ['ConfigurationItemChangeNotification', 'ScheduledNotification', 'OversizedConfigurationItemChangeNotification']: + configuration_item = get_configuration_item(invoking_event) + if is_applicable(configuration_item, event): + compliance_result = evaluate_compliance(event, configuration_item, valid_rule_parameters) + else: + compliance_result = "NOT_APPLICABLE" + else: + return build_internal_error_response('Unexpected message type', str(invoking_event)) + except botocore.exceptions.ClientError as ex: + if is_internal_error(ex): + return build_internal_error_response("Unexpected error while completing API request", str(ex)) + return build_error_response("Customer error while making API request", str(ex), ex.response['Error']['Code'], ex.response['Error']['Message']) + except ValueError as ex: + return build_internal_error_response(str(ex), str(ex)) + + evaluations = [] + latest_evaluations = [] + + if not compliance_result: + latest_evaluations.append(build_evaluation(event['accountId'], "NOT_APPLICABLE", event, resource_type='AWS::::Account')) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, str): + if configuration_item: + evaluations.append(build_evaluation_from_config_item(configuration_item, compliance_result)) + else: + evaluations.append(build_evaluation(event['accountId'], compliance_result, event, resource_type=DEFAULT_RESOURCE_TYPE)) + elif isinstance(compliance_result, list): + for evaluation in compliance_result: + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in evaluation: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + + if not missing_fields: + latest_evaluations.append(evaluation) + evaluations = clean_up_old_evaluations(latest_evaluations, event) + elif isinstance(compliance_result, dict): + missing_fields = False + for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'): + if field not in compliance_result: + print("Missing " + field + " from custom evaluation.") + missing_fields = True + if not missing_fields: + evaluations.append(compliance_result) + else: + evaluations.append(build_evaluation_from_config_item(configuration_item, 'NOT_APPLICABLE')) + + # Put together the request that reports the evaluation status + result_token = event['resultToken'] + test_mode = False + if result_token == 'TESTMODE': + # Used solely for RDK test to skip actual put_evaluation API call + test_mode = True + + # Invoke the Config API to report the result of the evaluation + evaluation_copy = [] + evaluation_copy = evaluations[:] + while evaluation_copy: + AWS_CONFIG_CLIENT.put_evaluations(Evaluations=evaluation_copy[:100], ResultToken=result_token, TestMode=test_mode) + del evaluation_copy[:100] + + # Used solely for RDK test to be able to test Lambda function + return evaluations + +def is_internal_error(exception): + return ((not isinstance(exception, botocore.exceptions.ClientError)) or exception.response['Error']['Code'].startswith('5') + or 'InternalError' in exception.response['Error']['Code'] or 'ServiceError' in exception.response['Error']['Code']) + +def build_internal_error_response(internal_error_message, internal_error_details=None): + return build_error_response(internal_error_message, internal_error_details, 'InternalError', 'InternalError') + +def build_error_response(internal_error_message, internal_error_details=None, customer_error_code=None, customer_error_message=None): + error_response = { + 'internalErrorMessage': internal_error_message, + 'internalErrorDetails': internal_error_details, + 'customerErrorMessage': customer_error_message, + 'customerErrorCode': customer_error_code + } + print(error_response) + return error_response diff --git a/python/GLUE_JOB_SECURITY_CONFIGURATION_ENCRYPTION/remediate_job_security_config_encryption.py b/python/GLUE_JOB_SECURITY_CONFIGURATION_ENCRYPTION/remediate_job_security_config_encryption.py new file mode 100644 index 00000000..22373b68 --- /dev/null +++ b/python/GLUE_JOB_SECURITY_CONFIGURATION_ENCRYPTION/remediate_job_security_config_encryption.py @@ -0,0 +1,132 @@ +import boto3, logging, sys +from botocore.exceptions import ClientError + +def lambda_handler(event, context): + print(event) + + glue_job_name = event['ResourceId'] + glue_client = boto3.client('glue') + + configuration_name = get_job_security_configuartion(glue_job_name, glue_client) + if not configuration_name: + logging.info('Glue job has no security configuration, creating a new one now') + new_security_configuration = create_security_configuration(glue_client, glue_job_name) + change_job_security_configuartion(glue_client, glue_job_name, new_security_configuration) + + else: #there is an existing configuration + #get encryption status dictionary + encryption_status = check_security_configuration(configuration_name, glue_client) + + #check s3 and cloudwatch encryption for security configuration + s3_encryption_config = encryption_status['S3Encryption'] + cloudwatch_encryption_config = encryption_status['CloudWatchEncryption'] + + #if encryption is not enabled for both, then create a new security configuration and update the glue job + if s3_encryption_config[next(iter(s3_encryption_config))] == 'DISABLED' or cloudwatch_encryption_config[next(iter(cloudwatch_encryption_config))] == 'DISABLED': + new_security_configuration = create_security_configuration(glue_client, glue_job_name) + change_job_security_configuartion(glue_client, glue_job_name, new_security_configuration) + logging.info(glue_job_name + "'s security configuration has been updated to a new one called " + new_security_configuration) + + else: logging.info('S3 and CloudWatch Encryption are both Enabled for ' + str(configuration_name)) + +def change_job_security_configuartion(glue_client, glue_job_name, new_security_configuration): + """This function updates the glue job's security configuration""" + + response = glue_client.get_job(JobName=glue_job_name)['Job'] + job_role = response['Role'] + job_command = response['Command'] + + glue_client.update_job( + JobName=glue_job_name, + JobUpdate= { + 'Command': job_command, + 'Role': job_role, + 'SecurityConfiguration': new_security_configuration + } + ) + +def check_security_configuration(security_configuration_name, glue_client): + """This function checks for S3 and Cloudwatch Encryption on the given security configuration and + returns a dictionary of the values""" + + response = glue_client.get_security_configuration( + Name=security_configuration_name + ) + + encryption_status = {} + encryption_config = response['SecurityConfiguration']['EncryptionConfiguration'] + + s3_encryption_config = encryption_config['S3Encryption'][0] + encryption_status['S3Encryption'] = s3_encryption_config + + cloudwatch_encryption_config = encryption_config['CloudWatchEncryption'] + encryption_status['CloudWatchEncryption'] = cloudwatch_encryption_config + + return encryption_status + +def get_job_security_configuartion(glue_job_name, glue_client): + """This function returns the security configuration name of the given glue job""" + response = glue_client.get_job( + JobName = glue_job_name + ) + try: + security_configuration = response['Crawler']['CrawlerSecurityConfiguration'] + except: + security_configuration = {} + + return security_configuration + + +def create_security_configuration(glue_client, glue_job_name): + """This function creates a new encrypted security configuration""" + + kms_key_arn = create_kms_key() + security_config_name = glue_job_name + '_SecurityConfiguration' + try: + response = glue_client.create_security_configuration( + Name= security_config_name, + EncryptionConfiguration={ + 'S3Encryption': [ + { + 'S3EncryptionMode': 'SSE-KMS', + 'KmsKeyArn': kms_key_arn + }, + ], + 'CloudWatchEncryption': { + 'CloudWatchEncryptionMode': 'SSE-KMS', + 'KmsKeyArn': kms_key_arn + } + }, + ) + return response['Name'] + + except ClientError as e: + if e.response['Error']['Code'] == 'AlreadyExistsException': + return security_config_name + else: + logging.critical('Security Configuration Creation Failed') + sys.exit(1) + + +def create_kms_key(): + """This function creates a symmetric KMS key and returns its ARN""" + kms_client = boto3.client('kms') + response = kms_client.create_key( + # Policy='string', + Description='This key was created to secure and encrypt AWS Glue', + KeyUsage='ENCRYPT_DECRYPT', #Alternative is 'SIGN_VERIFY'| + CustomerMasterKeySpec= 'SYMMETRIC_DEFAULT', #Alternatives are 'RSA_2048'|'RSA_3072'|'RSA_4096'|'ECC_NIST_P256'|'ECC_NIST_P384'|'ECC_NIST_P521'|'ECC_SECG_P256K1'|' + Origin='AWS_KMS', #Alternatives are |'EXTERNAL'|'AWS_CLOUDHSM', + # CustomKeyStoreId='string', + # BypassPolicyLockoutSafetyCheck=True|False, + Tags=[ + { + 'TagKey': 'string', + 'TagValue': 'string' + }, + ] + ) + + kms_key_arn = response['KeyMetadata']['Arn'] + logging.info('KMS KEY CREATED: ' + str(kms_key_arn)) + return kms_key_arn \ No newline at end of file