diff --git a/.travis.yml b/.travis.yml index e353738c2..7d964ab46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ matrix: install: - sed -i '/WTF_CSRF_ENABLED = True/c\WTF_CSRF_ENABLED = False' `pwd`/env-config/config.py - pip install bandit + - pip install pylint script: - coverage run -a -m py.test security_monkey/tests/auditors || exit 1 @@ -46,6 +47,7 @@ matrix: - coverage run -a -m py.test security_monkey/tests/interface || exit 1 - coverage run -a -m py.test security_monkey/tests/utilities || exit 1 - bandit -r -ll -ii -x security_monkey/tests . + - pylint -E -d E1101,E0611,F0401 --ignore=service.py,datastore.py,datastore_utils.py,watcher.py security_monkey after_success: - coveralls diff --git a/Dockerfile b/Dockerfile index f706537cf..b0c80c22e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ FROM ubuntu:14.04 MAINTAINER Netflix Open Source Development -ENV SECURITY_MONKEY_VERSION=v0.9.1 \ +ENV SECURITY_MONKEY_VERSION=v0.9.2 \ SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-docker.py RUN apt-get update &&\ diff --git a/dart/pubspec.yaml b/dart/pubspec.yaml index b6330c614..76c1aa8ed 100644 --- a/dart/pubspec.yaml +++ b/dart/pubspec.yaml @@ -1,6 +1,6 @@ name: security_monkey description: An AWS Policy Monitoring and Alerting Tool -version: 0.9.1 +version: 0.9.2 dependencies: angular: "^1.1.2+2" angular_ui: ">=0.6.8 <0.7.0" diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index f2aae08df..40746a91e 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -15,7 +15,7 @@ FROM nginx:1.11.4 MAINTAINER Netflix Open Source Development -ENV SECURITY_MONKEY_VERSION=v0.9.1 +ENV SECURITY_MONKEY_VERSION=v0.9.2 RUN apt-get update &&\ apt-get install -y curl git sudo apt-transport-https &&\ curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\ diff --git a/docs/changelog.md b/docs/changelog.md index da9aca645..749c975df 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,52 @@ Changelog ========= +v0.9.2 (2017-05-24) +---------------------------------------- + +- PR #695 - @mikegrima - Fixing jinja import bug affecting change emails. +- PR #692 - @LukeKennedy - Reduce number of API calls in Managed Policy watcher. +- PR #694 - @supertom - GCP Documentation Updates +- PR #701 - @supertom - Update GCP ServiceAccount Name to use email instead of DisplayName. +- PR #702 - @rodriguezsergio - Update KMS Auditor. Don't create issue when Effect is Deny for a wildcard principal. +- PR #697 - @mcpeak - Pylint fixes and TravisCI pylint enforcement. +- PR #706 - @monkeysecurity Fix bug where batched watchers did not send change alert emails. +- PR #708 - @redixin - Fix bug in docker config where `SECURITY_MONKEY_POSTGRES_PORT` would not work if passed as a string. +- PR #714 - @monkeysecurity - Fix bug where change emails from batched watchers had incorrect color in the JSON diff. +- PR #713 - @monkeysecurity - Fix path to favicon from flask-security jinja templates. +- PR #709 - @crruthe - Exempt SSO API from CSRF protection. +- PR #719 - @monkeysecurity - New simplified watcher format for CloudAux Technologies. +- PR #726 - @monkeysecurity, @willbengtson - Add new SAMLProvider watcher. +- PR #730 - @monkeysecurity - Fix bug where ephemerals were not respected for CloudAuxWatcher subclasses. +- PR #727 - @supertom - Fix bug where duplicate GCP names would violate DB's unique constraint. Names now contain project ID. +- PR #728 - @supertom - Basic Auditor Tests for GCP. +- @monkeysecurity - Updated link to Ubuntu's SSL documentation. +- @monkeysecurity - Bumped version of Cryptography dependency. +- PEP8 updates. + +Important Notes: +- Additional Permissions Required: + - "elasticloadbalancing:describelisteners", + - "elasticloadbalancing:describerules", + - "elasticloadbalancing:describesslpolicies", + - "elasticloadbalancing:describetags", + - "elasticloadbalancing:describetargetgroups", + - "elasticloadbalancing:describetargetgroupattributes", + - "elasticloadbalancing:describetargethealth", + - "iam:listsamlproviders", +- New Watcher: ALB (elbv2) +- ELB (v1) Watcher re-written with boto3 in CloudAux. Now respects the config value `SECURITYGROUP_INSTANCE_DETAIL` when determining whether to add the instance id's to the ELB definition. + +Contributors: +- @LukeKennedy +- @rodriguezsergio +- @redixin +- @crruthe +- @supertom +- @mcpeak +- @mikegrima +- @monkeysecurity + v0.9.1 (2017-04-20) ---------------------------------------- diff --git a/docs/iam_aws.md b/docs/iam_aws.md index fdc576759..1b415f999 100644 --- a/docs/iam_aws.md +++ b/docs/iam_aws.md @@ -98,6 +98,13 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly": "elasticloadbalancing:describeloadbalancerattributes", "elasticloadbalancing:describeloadbalancerpolicies", "elasticloadbalancing:describeloadbalancers", + "elasticloadbalancing:describelisteners", + "elasticloadbalancing:describerules", + "elasticloadbalancing:describesslpolicies", + "elasticloadbalancing:describetags", + "elasticloadbalancing:describetargetgroups", + "elasticloadbalancing:describetargetgroupattributes", + "elasticloadbalancing:describetargethealth", "es:describeelasticsearchdomainconfig", "es:listdomainnames", "iam:getaccesskeylastused", @@ -122,6 +129,7 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly": "iam:listpolicies", "iam:listrolepolicies", "iam:listroles", + "iam:listsamlproviders", "iam:listservercertificates", "iam:listsigningcertificates", "iam:listuserpolicies", diff --git a/docs/iam_gcp.md b/docs/iam_gcp.md index ff2334419..76999f226 100644 --- a/docs/iam_gcp.md +++ b/docs/iam_gcp.md @@ -30,6 +30,12 @@ To restrict which permissions Security Monkey has to your projects, we'll create ![Add User to Service Account](images/add_user_to_service_account.png "Add User to Service Account") +Enable IAM API +--------------- + +For each GCP project you would like Security Monkey to access, you'll need to enable the IAM API. Visit the [IAM API page](https://console.cloud.google.com/apis/api/iam.googleapis.com/overview) page in the web console + and click 'Enable API' at the top of the screen. When dealing with many projects, you might prefer to do this with the gcloud command. For details on how to enable services with gcloud, visit the + [service-management](https://cloud.google.com/service-management/enable-disable#enabling_services) page. The IAM service name is 'iam.googleapis.com'. Next: ----- diff --git a/docs/instance_launch_gcp.md b/docs/instance_launch_gcp.md index a16ff10e8..b9008c402 100644 --- a/docs/instance_launch_gcp.md +++ b/docs/instance_launch_gcp.md @@ -6,10 +6,11 @@ Create an instance running Ubuntu 14.04 LTS using our 'securitymonkey' service a Navigate to the [Create Instance page](https://console.developers.google.com/compute/instancesAdd). Fill in the following fields: - **Name**: securitymonkey -- **Zone**: If using GCP Cloud SQL, select the same zone here. +- **Zone**: If using GCP Cloud SQL, select the same zone here. [(Zone List)](https://cloud.google.com/compute/docs/regions-zones/regions-zones#available) - **Machine Type**: 1vCPU, 3.75GB (minimum; also known as n1-standard-1) - **Boot Disk**: Ubuntu 14.04 LTS - **Service Account**: securitymonkey +- **Firewall**: Allow HTTPS Traffic Click the *Create* button to create the instance. @@ -23,9 +24,8 @@ Connecting to your new instance: We will connect to the new instance over ssh with the gcloud command: - $ gcloud compute ssh @ --zone us-central + $ gcloud compute ssh securitymonkey --zone -Replace the first parameter `` with the username you authenticated gcloud with. Replace the last parameter `` with the Public IP of your instance. Next: ----- diff --git a/docs/quickstart.md b/docs/quickstart.md index c156ebab0..024b9f3ad 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -110,9 +110,9 @@ If you're using the bleeding edge (develop) branch, you will need to compile the /usr/lib/dart/bin/pub build # Copy the compiled Web UI to the appropriate destination - mkdir -p /usr/local/src/security_monkey/security_monkey/static/ - /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ - chgrp -R www-data /usr/local/src/security_monkey + sudo mkdir -p /usr/local/src/security_monkey/security_monkey/static/ + sudo /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ + sudo chgrp -R www-data /usr/local/src/security_monkey ### Configure the Application @@ -197,7 +197,7 @@ For this quickstart guide, we will use a self-signed SSL certificate. In product There are some great instructions for generating a certificate on the Ubuntu website: -[Ubuntu - Create a Self Signed SSL Certificate](https://help.ubuntu.com/12.04/serverguide/certificates-and-security.html) +[Ubuntu - Create a Self Signed SSL Certificate](https://help.ubuntu.com/14.04/serverguide/certificates-and-security.html) The last commands you need to run from that tutorial are in the "Installing the Certificate" section: diff --git a/env-config/config-docker.py b/env-config/config-docker.py index b188fbc5f..2e7c87df0 100644 --- a/env-config/config-docker.py +++ b/env-config/config-docker.py @@ -70,7 +70,7 @@ def env_to_bool(input): } } -SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%d/%s' % ( +SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % ( os.getenv('SECURITY_MONKEY_POSTGRES_USER', 'postgres'), os.getenv('SECURITY_MONKEY_POSTGRES_PASSWORD', 'securitymonkeypassword'), os.getenv('SECURITY_MONKEY_POSTGRES_HOST', 'localhost'), diff --git a/scripts/secmonkey_role_setup.py b/scripts/secmonkey_role_setup.py index a5e72ec1a..076ba2a30 100755 --- a/scripts/secmonkey_role_setup.py +++ b/scripts/secmonkey_role_setup.py @@ -87,6 +87,13 @@ "elasticloadbalancing:describeloadbalancerattributes", "elasticloadbalancing:describeloadbalancerpolicies", "elasticloadbalancing:describeloadbalancers", + "elasticloadbalancing:describelisteners", + "elasticloadbalancing:describerules", + "elasticloadbalancing:describesslpolicies", + "elasticloadbalancing:describetags", + "elasticloadbalancing:describetargetgroups", + "elasticloadbalancing:describetargetgroupattributes", + "elasticloadbalancing:describetargethealth", "es:describeelasticsearchdomainconfig", "es:listdomainnames", "iam:getaccesskeylastused", @@ -111,6 +118,7 @@ "iam:listpolicies", "iam:listrolepolicies", "iam:listroles", + "iam:listsamlproviders", "iam:listservercertificates", "iam:listsigningcertificates", "iam:listuserpolicies", diff --git a/security_monkey/__init__.py b/security_monkey/__init__.py index 0ce350879..b4d1dc656 100644 --- a/security_monkey/__init__.py +++ b/security_monkey/__init__.py @@ -23,7 +23,7 @@ import stat ### VERSION ### -__version__ = '0.9.1' +__version__ = '0.9.2' ### FLASK ### from flask import Flask diff --git a/security_monkey/auditors/elb.py b/security_monkey/auditors/elb.py index af8a43b5e..aabd18ae1 100644 --- a/security_monkey/auditors/elb.py +++ b/security_monkey/auditors/elb.py @@ -25,8 +25,10 @@ from security_monkey.datastore import NetworkWhitelistEntry from security_monkey.datastore import Item from security_monkey.watchers.security_group import SecurityGroup +from collections import defaultdict import ipaddr +import json # From https://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-security-policy-table.html DEPRECATED_CIPHERS = [ @@ -146,14 +148,14 @@ def check_internet_scheme(self, elb_item): """ alert when an ELB has an "internet-facing" scheme. """ - scheme = elb_item.config.get('scheme', None) - vpc = elb_item.config.get('vpc_id', None) + scheme = elb_item.config.get('Scheme', None) + vpc = elb_item.config.get('VPCId', None) if scheme and scheme == u"internet-facing" and not vpc: self.add_issue(1, 'ELB is Internet accessible.', elb_item) elif scheme and scheme == u"internet-facing" and vpc: # Grab each attached security group and determine if they contain # a public IP - security_groups = elb_item.config.get('security_groups', []) + security_groups = elb_item.config.get('SecurityGroups', []) sg_items = self.get_watcher_support_items(SecurityGroup.index, elb_item.account) for sgid in security_groups: for sg in sg_items: @@ -175,25 +177,35 @@ def check_internet_scheme(self, elb_item): def check_listener_reference_policy(self, elb_item): """ - alert when an SSL listener is not using the ELBSecurity Policy-2014-10 policy. + alert when an SSL listener is not using the latest reference policy. """ - listeners = elb_item.config.get('listeners') - for listener in listeners: - for policy in listener['policies']: - policy_type = policy.get("type", None) - if policy_type and policy_type == "SSLNegotiationPolicyType": - reference_policy = policy.get('reference_security_policy', None) - self._process_reference_policy(reference_policy, policy['name'], listener['load_balancer_port'], elb_item) - if not reference_policy: - self._process_custom_listener_policy(policy, listener['load_balancer_port'], elb_item) + policy_port_map = defaultdict(list) + for listener in elb_item.config.get('ListenerDescriptions'): + if len(listener.get('PolicyNames', [])) > 0: + for name in listener.get('PolicyNames', []): + policy_port_map[name].append(listener['LoadBalancerPort']) + + policies = elb_item.config.get('PolicyDescriptions') + for policy_name, policy in policies.items(): + policy_type = policy.get("type", None) + if policy_type and policy_type == "SSLNegotiationPolicyType": + reference_policy = policy.get('reference_security_policy', None) + self._process_reference_policy(reference_policy, policy_name, json.dumps(policy_port_map[policy_name]), elb_item) + if not reference_policy: + self._process_custom_listener_policy(policy_name, policy, json.dumps(policy_port_map[policy_name]), elb_item) def check_logging(self, elb_item): """ Alert when elb logging is not enabled """ - logging = elb_item.config.get('is_logging') + logging = elb_item.config.get('Attributes', {}).get('AccessLog', {}) if not logging: self.add_issue(1, 'ELB is not configured for logging.', elb_item) + return + + if not logging.get('Enabled'): + self.add_issue(1, 'ELB is not configured for logging.', elb_item) + return def _process_reference_policy(self, reference_policy, policy_name, port, elb_item): notes = "Policy {0} on port {1}".format(policy_name, port) @@ -266,7 +278,7 @@ def _process_reference_policy(self, reference_policy, policy_name, port, elb_ite notes = reference_policy self.add_issue(10, "Unknown reference policy.", elb_item, notes=notes) - def _process_custom_listener_policy(self, policy, port, elb_item): + def _process_custom_listener_policy(self, policy_name, policy, port, elb_item): """ Alerts on: sslv2 @@ -274,12 +286,12 @@ def _process_custom_listener_policy(self, policy, port, elb_item): missing server order preference deprecated ciphers """ - notes = "Policy {0} on port {1}".format(policy['name'], port) + notes = "Policy {0} on port {1}".format(policy_name, port) - if policy.get('sslv2', None): + if policy.get('protocols', {}).get('sslv2', None): self.add_issue(10, "SSLv2 is enabled", elb_item, notes=notes) - if policy.get('sslv3', None): + if policy.get('protocols', {}).get('sslv3', None): self.add_issue(10, "SSLv3 is enabled", elb_item, notes=notes) server_defined_cipher_order = policy.get('server_defined_cipher_order', None) diff --git a/security_monkey/auditors/kms.py b/security_monkey/auditors/kms.py index b183afb5a..9e1485e07 100644 --- a/security_monkey/auditors/kms.py +++ b/security_monkey/auditors/kms.py @@ -85,7 +85,7 @@ def check_for_kms_policy_with_foreign_account(self, kms_item): principal_account_ids = set() for arn in aws_principal: - if arn == "*" and not condition_accounts: + if arn == "*" and not condition_accounts and "allow" == statement.get('Effect').lower(): notes = "An KMS policy where { 'Principal': { 'AWS': '*' } } must also have" notes += " a {'Condition': {'StringEquals': { 'kms:CallerAccount': '' } } }" notes += " or it is open to the world." diff --git a/security_monkey/cloudaux_batched_watcher.py b/security_monkey/cloudaux_batched_watcher.py new file mode 100644 index 000000000..b122a4b76 --- /dev/null +++ b/security_monkey/cloudaux_batched_watcher.py @@ -0,0 +1,72 @@ +from security_monkey.cloudaux_watcher import CloudAuxWatcher +from security_monkey.cloudaux_watcher import CloudAuxChangeItem +from security_monkey.decorators import record_exception +from cloudaux.decorators import iter_account_region + + +class CloudAuxBatchedWatcher(CloudAuxWatcher): + + def __init__(self, **kwargs): + super(CloudAuxBatchedWatcher, self).__init__(**kwargs) + self.batched_size = 100 + self.done_slurping = False + + def slurp_list(self): + self.prep_for_batch_slurp() + + @record_exception(source='{index}-watcher'.format(index=self.index), pop_exception_fields=True) + def invoke_list_method(**kwargs): + return self.list_method(**kwargs['conn_dict']) + + @iter_account_region(self.service_name, accounts=self.account_identifiers, + regions=self._get_regions(), conn_type='dict') + def get_item_list(**kwargs): + kwargs, exception_map = self._add_exception_fields_to_kwargs(**kwargs) + items = invoke_list_method(**kwargs) + + if not items: + self.done_slurping = True + items = list() + + return items, exception_map + + items, exception_map = self._flatten_iter_response(get_item_list()) + self.total_list.extend(items) + + return items, exception_map + + def slurp(self): + + @record_exception(source='{index}-watcher'.format(index=self.index), pop_exception_fields=True) + def invoke_get_method(item, **kwargs): + return self.get_method(item, **kwargs['conn_dict']) + + @iter_account_region(self.service_name, accounts=self.account_identifiers, + regions=self._get_regions(), conn_type='dict') + def slurp_items(**kwargs): + item_list = list() + kwargs, exception_map = self._add_exception_fields_to_kwargs(**kwargs) + item_counter = self.batch_counter * self.batched_size + while self.batched_size - len(item_list) > 0 and not self.done_slurping: + cursor = self.total_list[item_counter] + item_name = self.get_name_from_list_output(cursor) + if item_name and self.check_ignore_list(item_name): + item_counter += 1 + if item_counter == len(self.total_list): + self.done_slurping = True + continue + + item_details = invoke_get_method(cursor, name=item_name, **kwargs) + if item_details: + item = CloudAuxChangeItem.from_item( + name=item_name, + item=item_details, + override_region=self.override_region, **kwargs) + item_list.append(item) + item_counter += 1 + if item_counter == len(self.total_list): + self.done_slurping = True + self.batch_counter += 1 + return item_list, exception_map + + return self._flatten_iter_response(slurp_items()) diff --git a/security_monkey/cloudaux_watcher.py b/security_monkey/cloudaux_watcher.py new file mode 100644 index 000000000..e3637b695 --- /dev/null +++ b/security_monkey/cloudaux_watcher.py @@ -0,0 +1,120 @@ +from security_monkey.watcher import Watcher, ChangeItem +from security_monkey.decorators import record_exception +from cloudaux.decorators import iter_account_region + + +class CloudAuxWatcher(Watcher): + index = 'abstract' + i_am_singular = 'Abstract Watcher' + i_am_plural = 'Abstract Watchers' + honor_ephemerals = False + ephemeral_paths = list() + override_region = None + service_name = None + def list_method(self, **kwargs): raise Exception('Not Implemented') + def get_method(self, item, **kwargs): raise Exception('Not Implemented') + def get_name_from_list_output(self, item): return item['Name'] + + def __init__(self, accounts=None, debug=None): + super(CloudAuxWatcher, self).__init__(accounts=accounts, debug=debug) + + def _get_account_name(self, identifier): + idx = 0 + for ident in self.account_identifiers: + if ident == identifier: + return self.accounts[idx] + + def _get_assume_role(self, identifier): + from security_monkey.datastore import Account + account = Account.query.filter(Account.identifier == identifier).first() + return account.getCustom("role_name") or 'SecurityMonkey' + + def _get_regions(self): + from security_monkey.decorators import get_regions + from security_monkey.datastore import Account + # pick an arbitrary account: + identifier = self.account_identifiers[0] + account = Account.query.filter(Account.identifier == identifier).first() + _, regions = get_regions(account, self.service_name) + return regions + + def _add_exception_fields_to_kwargs(self, **kwargs): + exception_map = dict() + kwargs['index'] = self.index + kwargs['account_name'] = self._get_account_name(kwargs['conn_dict']['account_number']) + kwargs['exception_record_region'] = self.override_region or kwargs['conn_dict']['region'] + kwargs['exception_map'] = exception_map + kwargs['conn_dict']['assume_role'] = self._get_assume_role(kwargs['conn_dict']['account_number']) + del kwargs['conn_dict']['tech'] + del kwargs['conn_dict']['service_type'] + return kwargs, exception_map + + def _flatten_iter_response(self, response): + """ + The cloudaux iter_account_region decorator returns a list of tuples. + Each tuple contains two members. 1) The result. 2) The exception map. + This method combines that list of tuples into a single result list and a single exception map. + """ + items = list() + exception_map = dict() + for result in response: + items.extend(result[0]) + exception_map.update(result[1]) + return items, exception_map + + def slurp(self): + self.prep_for_slurp() + + @record_exception(source='{index}-watcher'.format(index=self.index), pop_exception_fields=True) + def invoke_list_method(**kwargs): + return self.list_method(**kwargs['conn_dict']) + + @record_exception(source='{index}-watcher'.format(index=self.index), pop_exception_fields=True) + def invoke_get_method(item, **kwargs): + return self.get_method(item, **kwargs['conn_dict']) + + @iter_account_region(self.service_name, accounts=self.account_identifiers, + regions=self._get_regions(), conn_type='dict') + def slurp_items(**kwargs): + kwargs, exception_map = self._add_exception_fields_to_kwargs(**kwargs) + + results = [] + item_list = invoke_list_method(**kwargs) + if not item_list: + return results, exception_map + + for item in item_list: + item_name = self.get_name_from_list_output(item) + if item_name and self.check_ignore_list(item_name): + continue + + item_details = invoke_get_method(item, name=item_name, **kwargs) + if item_details: + item = CloudAuxChangeItem.from_item( + name=item_name, + item=item_details, + override_region=self.override_region, **kwargs) + results.append(item) + + return results, exception_map + return self._flatten_iter_response(slurp_items()) + +class CloudAuxChangeItem(ChangeItem): + def __init__(self, index=None, account=None, region='us-east-1', name=None, arn=None, config={}): + super(CloudAuxChangeItem, self).__init__( + index=index, + region=region, + account=account, + name=name, + arn=arn, + new_config=config) + + @classmethod + def from_item(cls, name, item, override_region, **kwargs): + return cls( + name=name, + arn=item['Arn'], + account=kwargs.get('account_name', kwargs.get('ProjectId')), + index=kwargs['index'], + region=override_region or kwargs['conn_dict']['region'], + config=item) diff --git a/security_monkey/common/arn.py b/security_monkey/common/arn.py index 34308d1c9..b0fae5621 100644 --- a/security_monkey/common/arn.py +++ b/security_monkey/common/arn.py @@ -40,15 +40,18 @@ def __init__(self, input): if arn_match.group(2) == "iam" and arn_match.group(5) == "root": self.root = True - return self._from_arn(arn_match, input) + self._from_arn(arn_match, input) + return acct_number_match = re.search('^(\d{12})+$', input) if acct_number_match: - return self._from_account_number(input) + self._from_account_number(input) + return aws_service_match = re.search('^([^.]*)\.amazonaws\.com$', input) if aws_service_match: - return self._from_aws_service(input, aws_service_match.group(1)) + self._from_aws_service(input, aws_service_match.group(1)) + return self.error = True app.logger.warn('ARN Could not parse [{}].'.format(input)) diff --git a/security_monkey/common/gcp/util.py b/security_monkey/common/gcp/util.py index eff7e40c7..d31a7cb23 100644 --- a/security_monkey/common/gcp/util.py +++ b/security_monkey/common/gcp/util.py @@ -54,12 +54,14 @@ def get_gcp_project_creds(account_names): return project_creds -def gcp_resource_id_builder(service, identifier, region=''): - resource = 'gcp:%s:%s:%s' % (region, service, identifier) +def gcp_resource_id_builder(service, identifier, project_id, region=''): + resource = 'gcp:%s:%s:%s:%s' % (project_id, region, service, identifier) return resource.replace('/', ':').replace('.', ':') -def modify(d, format='camelized'): - return cloudaux_modify(d, format=format) + +def modify(d, output='camelized'): + return cloudaux_modify(d, output=format) + def get_user_agent(**kwargs): from security_monkey.common.gcp.config import ApplicationConfig as appconfig diff --git a/security_monkey/common/jinja.py b/security_monkey/common/jinja.py index 51578c3a4..c76848753 100644 --- a/security_monkey/common/jinja.py +++ b/security_monkey/common/jinja.py @@ -31,8 +31,7 @@ def get_jinja_env(): Returns a Jinja environment with a FileSystemLoader for our templates """ templates_directory = os.path.abspath(os.path.join(__file__, '..', '..', templates)) - jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_directory), # nosec - autoescape=select_autoescape(['html', 'xml'])) - # nosec - jinja autoescape enabled for potentially dangerous extensions html, xml + jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_directory)) # nosec + # templates are HTML escaped elsewhere #jinja_environment.filters['dateformat'] = dateformat return jinja_environment diff --git a/security_monkey/common/route53.py b/security_monkey/common/route53.py index 7a0238610..737509109 100644 --- a/security_monkey/common/route53.py +++ b/security_monkey/common/route53.py @@ -25,6 +25,7 @@ import boto.route53.record from security_monkey import app +from security_monkey.exceptions import ZoneIDNotFound class Route53Service(object): diff --git a/security_monkey/datastore_utils.py b/security_monkey/datastore_utils.py index 9b5afed6b..915b27a49 100644 --- a/security_monkey/datastore_utils.py +++ b/security_monkey/datastore_utils.py @@ -101,25 +101,25 @@ def detect_change(item, account, technology, complete_hash, durable_hash): app.logger.debug("Couldn't find item: {tech}/{account}/{region}/{item} in DB.".format( tech=item.index, account=item.account, region=item.region, item=item.name )) - return True, 'durable', result + return True, 'durable', result, 'created' if result.latest_revision_durable_hash != durable_hash: app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has a DURABLE CHANGE.".format( tech=item.index, account=item.account, region=item.region, item=item.name )) - return True, 'durable', result + return True, 'durable', result, 'changed' elif result.latest_revision_complete_hash != complete_hash: app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has an EPHEMERAL CHANGE.".format( tech=item.index, account=item.account, region=item.region, item=item.name )) - return True, 'ephemeral', result + return True, 'ephemeral', result, None else: app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has NO CHANGE.".format( tech=item.index, account=item.account, region=item.region, item=item.name )) - return False, None, result + return False, None, result, None def result_from_item(item, account, technology): @@ -161,12 +161,6 @@ def inactivate_old_revisions(watcher, arns, account, technology): db_item.latest_revision_id = revision.id datastore.db.session.add(db_item) - # Find any audit issues associated with this revision, and delete it: - ia = ItemAudit.query.filter(ItemAudit.item_id == db_item.id).all() - - for audit_item in ia: - datastore.db.session.delete(audit_item) - datastore.db.session.commit() return result diff --git a/security_monkey/decorators.py b/security_monkey/decorators.py index dbd62f462..c7c3aea4e 100644 --- a/security_monkey/decorators.py +++ b/security_monkey/decorators.py @@ -127,7 +127,7 @@ def decorated_function(*args, **kwargs): for account_name in accounts: account = Account.query.filter(Account.name == account_name).first() if not account: - app.logger.error("Couldn't find account with name", account_name) + app.logger.error("Couldn't find account with name {}".format(account_name)) return try: diff --git a/security_monkey/exceptions.py b/security_monkey/exceptions.py index 2e02cfdcd..e8a1235b8 100644 --- a/security_monkey/exceptions.py +++ b/security_monkey/exceptions.py @@ -119,3 +119,12 @@ def __init__(self, account_name): def __str__(self): return repr("Account with name: {} already exists. Cannnot create" " or rename account with this name.".format(self.account_name)) + +class ZoneIDNotFound(SecurityMonkeyException): + """Zone ID is not found during lookup""" + def __init__(self, domain): + self.domain = domain + app.logger.error(self) + + def __str__(self): + return repr("Given domain ({}) not found in hosted zones".format(self.domain)) diff --git a/security_monkey/jirasync.py b/security_monkey/jirasync.py index 7b8039ba2..147ca977e 100644 --- a/security_monkey/jirasync.py +++ b/security_monkey/jirasync.py @@ -55,7 +55,7 @@ def __init__(self, jira_file): elif (self.ip_proxy is None and self.port_proxy): app.logger.warn("Proxy port set, but not proxy host. Skipping JIRA proxy settings.") - self.client = JIRA(self.server, basic_auth=(self.account, self.password), options=options, proxies=proxies) + self.client = JIRA(self.server, basic_auth=(self.account, self.password), options=options, proxies=proxies) # pylint: disable=E1123 except Exception as e: raise Exception("Error connecting to JIRA: {}".format(str(e)[:1024])) diff --git a/security_monkey/monitors.py b/security_monkey/monitors.py index 32b784ee2..7c82d7336 100644 --- a/security_monkey/monitors.py +++ b/security_monkey/monitors.py @@ -10,6 +10,7 @@ from security_monkey.auditor import auditor_registry from security_monkey.watcher import watcher_registry from security_monkey.account_manager import account_registry, get_account_by_name +from security_monkey import app class Monitor(object): @@ -102,7 +103,7 @@ def _set_dependency_hierarchies(monitor_dict, monitor, path, level): support_mon = monitor_dict.get(support_index) if support_mon == None: app.logger.warn("Monitor {0} depends on monitor {1}, but {1} is unavailable" - .format(monitor.watcher.index, support_index)) + .format(monitor.watcher.index, support_index)) else: if support_mon.audit_tier < level: support_mon.audit_tier = level diff --git a/security_monkey/scheduler.py b/security_monkey/scheduler.py index 21c3ccd52..56efdfd68 100644 --- a/security_monkey/scheduler.py +++ b/security_monkey/scheduler.py @@ -60,7 +60,7 @@ def find_changes(accounts, monitor_names, debug=True): def batch_logic(monitor, current_watcher, account_name, debug): # Fetch the full list of items that we need to obtain: - exception_map = current_watcher.slurp_list() + _, exception_map = current_watcher.slurp_list() if len(exception_map) > 0: # Get the location tuple to collect the region: location = exception_map.keys()[0] diff --git a/security_monkey/sso/views.py b/security_monkey/sso/views.py index c41fabad3..544db9e50 100644 --- a/security_monkey/sso/views.py +++ b/security_monkey/sso/views.py @@ -25,11 +25,13 @@ from .service import fetch_token_header_payload, get_rsa_public_key, setup_user from security_monkey.datastore import User -from security_monkey import db, rbac +from security_monkey import db, rbac, csrf from urlparse import urlparse mod = Blueprint('sso', __name__) +# SSO providers implement their own CSRF protection +csrf.exempt(mod) api = Api(mod) diff --git a/security_monkey/templates/security/forgot_password.html b/security_monkey/templates/security/forgot_password.html index 1770c42e0..30c3773fd 100644 --- a/security_monkey/templates/security/forgot_password.html +++ b/security_monkey/templates/security/forgot_password.html @@ -7,7 +7,7 @@ - + Security Monkey diff --git a/security_monkey/templates/security/login_user.html b/security_monkey/templates/security/login_user.html index 111656b29..8a809f93e 100644 --- a/security_monkey/templates/security/login_user.html +++ b/security_monkey/templates/security/login_user.html @@ -7,7 +7,7 @@ - + Security Monkey diff --git a/security_monkey/templates/security/register_user.html b/security_monkey/templates/security/register_user.html index 6014a8e8f..3e3dc72ad 100644 --- a/security_monkey/templates/security/register_user.html +++ b/security_monkey/templates/security/register_user.html @@ -7,7 +7,7 @@ - + Security Monkey diff --git a/security_monkey/templates/security/reset_password.html b/security_monkey/templates/security/reset_password.html index 37cc6ab0c..99ccf1332 100644 --- a/security_monkey/templates/security/reset_password.html +++ b/security_monkey/templates/security/reset_password.html @@ -7,7 +7,7 @@ - + Security Monkey diff --git a/security_monkey/templates/security/send_confirmation.html b/security_monkey/templates/security/send_confirmation.html index 201c320af..4ec0e826a 100644 --- a/security_monkey/templates/security/send_confirmation.html +++ b/security_monkey/templates/security/send_confirmation.html @@ -7,7 +7,7 @@ - + Security Monkey diff --git a/security_monkey/tests/auditors/gcp/__init__.py b/security_monkey/tests/auditors/gcp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/security_monkey/tests/auditors/gcp/gce/test_firewall.py b/security_monkey/tests/auditors/gcp/gce/test_firewall.py new file mode 100644 index 000000000..95ffcce09 --- /dev/null +++ b/security_monkey/tests/auditors/gcp/gce/test_firewall.py @@ -0,0 +1,105 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +""" +.. module: security_monkey.tests.auditors.gcp.gce.test_firewall + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Tom Melendez @supertom +""" + +ALLOWED_LIST_WITH_PORTRANGE = [ + { + "IPProtocol": "tcp", + "ports": [ + "0-65535" + ] + }, + { + "IPProtocol": "udp", + "ports": [ + "0-65535" + ] + }, + { + "IPProtocol": "icmp" + } + ] + +ALLOWED_LIST_NO_PORTRANGE = [ + { + "IPProtocol": "tcp", + "ports": [ + "80" + ] + }, + { + "IPProtocol": "icmp" + } + ] + +SOURCERANGES_PRESENT = [ + '0.0.0.0/0', + '192.168.1.0/24' + ] + +SOURCERANGES_ABSENT = [ + '10.0.0.0/0', + '192.168.1.0/24' + ] + +TARGETTAGS_PRESENT = [ + 'http-server', + 'https-server' + ] + +TARGETTAGS_ABSENT = [ + 'http-server', + 'https-server' + ] + +class FirewallTestCase(unittest.TestCase): + + def test___port_range_exists(self): + from security_monkey.auditors.gcp.gce.firewall import GCEFirewallRuleAuditor + auditor = GCEFirewallRuleAuditor(accounts=['unittest']) + actual = auditor._port_range_exists(ALLOWED_LIST_WITH_PORTRANGE) + self.assertTrue(isinstance(actual, list)) + + actual = auditor._port_range_exists(ALLOWED_LIST_NO_PORTRANGE) + self.assertFalse(actual) + + def test__target_tags_valid(self): + from security_monkey.auditors.gcp.gce.firewall import GCEFirewallRuleAuditor + auditor = GCEFirewallRuleAuditor(accounts=['unittest']) + + actual = auditor._target_tags_valid(TARGETTAGS_PRESENT) + self.assertTrue(isinstance(actual, list)) + + actual = auditor._target_tags_valid(TARGETTAGS_ABSENT) + self.assertFalse(actual) + + def test__source_ranges_open(self): + from security_monkey.auditors.gcp.gce.firewall import GCEFirewallRuleAuditor + auditor = GCEFirewallRuleAuditor(accounts=['unittest']) + + actual = auditor._source_ranges_open(SOURCERANGES_PRESENT) + self.assertTrue(isinstance(actual, list)) + + actual = auditor._source_ranges_open(SOURCERANGES_ABSENT) + self.assertFalse(actual) + +if __name__ == '__main__': + unittest.main() diff --git a/security_monkey/tests/auditors/gcp/gce/test_network.py b/security_monkey/tests/auditors/gcp/gce/test_network.py new file mode 100644 index 000000000..9129d2ad6 --- /dev/null +++ b/security_monkey/tests/auditors/gcp/gce/test_network.py @@ -0,0 +1,71 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +""" +.. module: security_monkey.tests.auditors.gcp.gce.test_network + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Tom Melendez @supertom +""" + +AUTO_NETWORK_DICT = { + "AutoCreateSubnetworks": True, + "CreationTimestamp": "2016-05-09T11:15:47.434-07:00", + "Description": "Default network for the project", + "Id": "5748637682906434876", + "Kind": "compute#network", + "Name": "default", + "SelfLink": "https://www.googleapis.com/compute/v1/projects/my-project-one/global/networks/default", + "Subnetworks": [ + { + "CreationTimestamp": "2016-10-25T09:53:00.777-07:00", + "GatewayAddress": "10.146.0.1", + "Id": "1852214226435846915", + "IpCidrRange": "10.146.0.0/20", + "Kind": "compute#subnetwork", + "Name": "default", + "Network": "https://www.googleapis.com/compute/v1/projects/my-project-one/global/networks/default", + "Region": "https://www.googleapis.com/compute/v1/projects/my-project-one/regions/asia-northeast1", + "SelfLink": "https://www.googleapis.com/compute/v1/projects/my-project-one/regions/asia-northeast1/subnetworks/default" + } + + ] +} + +LEGACY_NETWORK = { + "kind": "compute#network", + "id": "6570954185952335682", + "creationTimestamp": "2016-08-04T13:46:37.261-07:00", + "name": "network-b", + "IPv4Range": "10.0.0.0/8", + "gatewayIPv4": "10.0.0.1", + "selfLink": "https://www.googleapis.com/compute/v1/projects/supertom-graphite/global/networks/network-b" +} + + + +class NetworkTestCase(unittest.TestCase): + + def test__legacy_exists(self): + from security_monkey.auditors.gcp.gce.network import GCENetworkAuditor + auditor = GCENetworkAuditor(accounts=['unittest']) + + actual = auditor._legacy_exists(AUTO_NETWORK_DICT) + self.assertFalse(actual) + actual = auditor._legacy_exists(LEGACY_NETWORK) + self.assertTrue(isinstance(actual, list)) + +if __name__ == '__main__': + unittest.main() diff --git a/security_monkey/tests/auditors/gcp/gcs/test_bucket.py b/security_monkey/tests/auditors/gcp/gcs/test_bucket.py new file mode 100644 index 000000000..0c2a68b49 --- /dev/null +++ b/security_monkey/tests/auditors/gcp/gcs/test_bucket.py @@ -0,0 +1,66 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +""" +.. module: security_monkey.tests.auditors.gcp.gcs.test_bucket + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Tom Melendez @supertom +""" + +ACL_LIST = [ + {u'role': u'OWNER', u'entity': u'project-editors-2094195755359'}, + {u'role': u'READER', u'entity': u'project-viewers-2094195755359'}, + {u'role': u'WRITER', u'entity': u'project-writer-2094195755359'} +] + +ACL_LIST_TWO_OWNERS = [ + {u'role': u'OWNER', u'entity': u'project-editors-2094195755359'}, + {u'role': u'READER', u'entity': u'project-viewers-2094195755359'}, + {u'role': u'OWNER', u'entity': u'project-editors-2094195755359'} +] + +ACL_LIST_ALLUSERS = [ + {u'role': u'OWNER', u'entity': u'project-editors-2094195755359'}, + {u'role': u'READER', u'entity': u'allUsers'}, + {u'role': u'OWNER', u'entity': u'project-editors-2094195755359'} +] + + +class BucketTestCase(unittest.TestCase): + + def test__acl_allusers_exists(self): + from security_monkey.auditors.gcp.gcs.bucket import GCSBucketAuditor + auditor = GCSBucketAuditor(accounts=['unittest']) + + actual = auditor._acl_allusers_exists(ACL_LIST) + self.assertFalse(actual) + actual = auditor._acl_allusers_exists(ACL_LIST_ALLUSERS) + self.assertTrue(actual) + + def test__acl_max_owners(self): + from security_monkey.auditors.gcp.gcs.bucket import GCSBucketAuditor + auditor = GCSBucketAuditor(accounts=['unittest']) + + # NOTE: the config value below actually controls this so ensure + # it is set to 1 + auditor.gcp_config.MAX_OWNERS_PER_BUCKET = 1 + actual = auditor._acl_max_owners(ACL_LIST) + self.assertFalse(actual) + actual = auditor._acl_max_owners(ACL_LIST_TWO_OWNERS) + self.assertTrue(actual) + +if __name__ == '__main__': + unittest.main() diff --git a/security_monkey/tests/auditors/gcp/iam/test_serviceaccount.py b/security_monkey/tests/auditors/gcp/iam/test_serviceaccount.py new file mode 100644 index 000000000..02ab6cce3 --- /dev/null +++ b/security_monkey/tests/auditors/gcp/iam/test_serviceaccount.py @@ -0,0 +1,70 @@ +# Copyright 2017 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +""" +.. module: security_monkey.tests.auditors.gcp.iam.test_serviceaccount + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Tom Melendez @supertom +""" + +POLICY_WITH_ACTOR_LIST = [ + { + "Members": [ + "user:test-user@gmail.com" + ], + "Role": "roles/iam.serviceAccountActor" + } +] + + +POLICY_NO_ACTOR_LIST = [ + { + "Members": [ + "user:test-user@gmail.com" + ], + "Role": "roles/viewer" + } +] + + +class IAMServiceAccountTestCase(unittest.TestCase): + + def test__max_keys(self): + from security_monkey.auditors.gcp.iam.serviceaccount import IAMServiceAccountAuditor + auditor = IAMServiceAccountAuditor(accounts=['unittest']) + # NOTE: the config value below actually controls this so ensure + # it is set to 1 + auditor.gcp_config.MAX_SERVICEACCOUNT_KEYS = 1 + actual = auditor._max_keys(2) + self.assertTrue(isinstance(actual, list)) + + actual = auditor._max_keys(1) + self.assertFalse(actual) + + def test__actor_role(self): + from security_monkey.auditors.gcp.iam.serviceaccount import IAMServiceAccountAuditor + auditor = IAMServiceAccountAuditor(accounts=['unittest']) + # NOTE: the config value below actually controls this so ensure + # it is set to 1 + auditor.gcp_config.MAX_SERVICEACCOUNT_KEYS = 1 + actual = auditor._actor_role(POLICY_WITH_ACTOR_LIST) + self.assertTrue(isinstance(actual, list)) + + actual = auditor._actor_role(POLICY_NO_ACTOR_LIST) + self.assertFalse(actual) + +if __name__ == '__main__': + unittest.main() diff --git a/security_monkey/tests/auditors/test_elb.py b/security_monkey/tests/auditors/test_elb.py new file mode 100644 index 000000000..316e3621c --- /dev/null +++ b/security_monkey/tests/auditors/test_elb.py @@ -0,0 +1,420 @@ +from security_monkey.datastore import Account, AccountType +from security_monkey.tests import SecurityMonkeyTestCase +from security_monkey import db + + +INTERNET_ELB = { + "Arn": "arn:aws:elasticloadbalancing:us-east-1:012345678901:loadbalancer/MyELB", + "Attributes": { + "ConnectionDraining": { + "Enabled": False, + "Timeout": 300 + }, + "CrossZoneLoadBalancing": { + "Enabled": False + }, + "ConnectionSettings": { + "IdleTimeout": 60 + }, + "AccessLog": { + "S3BucketPrefix": "test", + "Enabled": True, + "EmitInterval": 5, + "S3BucketName": "some-log-bucket" + } + }, + "AvailabilityZones": [ + "us-east-1b" + ], + "BackendServerDescriptions": [], + "CanonicalHostedZoneNameID": "Z3DZXE0Q79N41H", + "CreatedTime": "2015-07-07 19:15:06.490000+00:00", + "DNSName": "MyELB-1885487881.us-east-1.elb.amazonaws.com", + "HealthCheck": { + "HealthyThreshold": 2, + "Interval": 30, + "Target": "HTTP:5050/health", + "Timeout": 5, + "UnhealthyThreshold": 2 + }, + "Instances": [], + "ListenerDescriptions": [ + { + "InstancePort": 80, + "PolicyNames": [], + "LoadBalancerPort": 80, + "Protocol": "HTTP", + "InstanceProtocol": "HTTP" + }, + { + "InstancePort": 2181, + "PolicyNames": [], + "LoadBalancerPort": 2181, + "Protocol": "TCP", + "InstanceProtocol": "TCP" + }, + { + "InstancePort": 5050, + "PolicyNames": [], + "LoadBalancerPort": 5050, + "Protocol": "HTTP", + "InstanceProtocol": "HTTP" + }, + { + "InstancePort": 8080, + "PolicyNames": [], + "LoadBalancerPort": 8080, + "Protocol": "HTTP", + "InstanceProtocol": "HTTP" + }, + { + "InstancePort": 8181, + "PolicyNames": [], + "LoadBalancerPort": 8181, + "Protocol": "HTTP", + "InstanceProtocol": "HTTP" + }, + { + "InstancePort": 80, + "Protocol": "HTTPS", + "InstanceProtocol": "HTTP", + "LoadBalancerPort": 443, + "PolicyNames": [ + "ELBSecurityPolicy-2016-08" + ], + "SSLCertificateId": "arn:aws:iam::012345678901:server-certificate/somecert-20170511-20180511" + } + ], + "LoadBalancerName": "MyELB", + "Policies": { + "LBCookieStickinessPolicies": [], + "AppCookieStickinessPolicies": [], + "OtherPolicies": [ + "ELBSecurityPolicy-2016-08" + ] + }, + "PolicyDescriptions": { + "ELBSecurityPolicy-2016-08": { + "reference_security_policy": "ELBSecurityPolicy-2016-08", + "supported_ciphers": [ + "AES128-GCM-SHA256", + "AES128-SHA", + "AES128-SHA256", + "AES256-GCM-SHA384", + "AES256-SHA", + "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA256", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-SHA", + "ECDHE-RSA-AES256-SHA384" + ], + "type": "SSLNegotiationPolicyType", + "server_defined_cipher_order": True, + "protocols": { + "tlsv1": True, + "tlsv1_1": True, + "tlsv1_2": True, + "sslv3": True, + "sslv2": False + } + } + }, + "Region": "us-east-1", + "Scheme": "internet-facing", + "SecurityGroups": [ + "sg-12345678" + ], + "SourceSecurityGroup": { + "OwnerAlias": "012345678901", + "GroupName": "MySG" + }, + "Subnets": [ + "subnet-19999999" + ], + "Tags": [ + { + "Value": "arn:aws:cloudformation:us-east-1:012345678901:stack/STACK/xxxxxxxxxxxxxxxxxxx", + "Key": "aws:cloudformation:stack-id" + } + ], + "VPCId": "vpc-49999999", + "_version": 2 +} + +INTERNET_SG = { + 'id': 'sg-12345678', + 'name': 'INTERNETSG', + 'rules': [ + { + 'cidr_ip': '0.0.0.0/0', + 'rule_type': 'ingress' + } + ] +} + +INTERNAL_SG = { + 'id': 'sg-87654321', + 'name': 'INTERNALSG', + 'rules': [ + { + 'cidr_ip': '10.0.0.0/8', + 'rule_type': 'ingress' + } + ] +} + + +class ELBTestCase(SecurityMonkeyTestCase): + def pre_test_setup(self): + account_type_result = AccountType(name='AWS') + db.session.add(account_type_result) + db.session.commit() + + account = Account(identifier="012345678910", name="TEST_ACCOUNT", + account_type_id=account_type_result.id, notes="TEST_ACCOUNT", + third_party=False, active=True) + + db.session.add(account) + db.session.commit() + + def test_check_internet_scheme_internet(self): + # internet-facing + # 0.0.0.0/0 + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + def mock_get_watcher_support_items(*args, **kwargs): + from security_monkey.watchers.security_group import SecurityGroupItem + sg_item = SecurityGroupItem(region='us-east-1', account='TEST_ACCOUNT', name='INTERNETSG', config=INTERNET_SG) + return [sg_item] + + auditor.get_watcher_support_items = mock_get_watcher_support_items + + auditor.check_internet_scheme(item) + + self.assertEqual(len(item.audit_issues), 1) + issue = item.audit_issues[0] + self.assertEqual(issue.issue, 'VPC ELB is Internet accessible.') + self.assertEqual(issue.notes, 'SG [INTERNETSG] via [0.0.0.0/0]') + + def test_check_internet_scheme_internet_2(self): + # internet-facing + # 10.0.0.0/8 + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + def mock_get_watcher_support_items(*args, **kwargs): + from security_monkey.watchers.security_group import SecurityGroupItem + sg_item = SecurityGroupItem(region='us-east-1', account='TEST_ACCOUNT', name='INTERNETSG', config=INTERNAL_SG) + return [sg_item] + + auditor.get_watcher_support_items = mock_get_watcher_support_items + + auditor.check_internet_scheme(item) + + self.assertEqual(len(item.audit_issues), 0) + + def test_check_internet_scheme_internal(self): + # internal + # 10.0.0.0/8 + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + INTERNAL_ELB = dict(INTERNET_ELB) + INTERNAL_ELB['Scheme'] = 'internal' + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNAL_ELB) + + def mock_get_watcher_support_items(*args, **kwargs): + from security_monkey.watchers.security_group import SecurityGroupItem + sg_item = SecurityGroupItem(region='us-east-1', account='TEST_ACCOUNT', name='INTERNETSG', config=INTERNAL_SG) + return [sg_item] + + auditor.get_watcher_support_items = mock_get_watcher_support_items + + auditor.check_internet_scheme(item) + + self.assertEqual(len(item.audit_issues), 0) + + def test_check_internet_scheme_internal_2(self): + # internal + # 0.0.0.0/0 + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + INTERNAL_ELB = dict(INTERNET_ELB) + INTERNAL_ELB['Scheme'] = 'internal' + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNAL_ELB) + + def mock_get_watcher_support_items(*args, **kwargs): + from security_monkey.watchers.security_group import SecurityGroupItem + sg_item = SecurityGroupItem(region='us-east-1', account='TEST_ACCOUNT', name='INTERNETSG', config=INTERNET_SG) + return [sg_item] + + auditor.get_watcher_support_items = mock_get_watcher_support_items + + auditor.check_internet_scheme(item) + + self.assertEqual(len(item.audit_issues), 0) + + def test_process_reference_policy(self): + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + auditor._process_reference_policy(None, 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 1) + self.assertEqual(item.audit_issues[0].issue, 'Custom listener policies are discouraged.') + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2011-08', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 5) + issues = [issue.issue for issue in item.audit_issues] + self.assertIn("ELBSecurityPolicy-2011-08 is vulnerable and deprecated", issues) + self.assertIn("ELBSecurityPolicy-2011-08 is vulnerable to poodlebleed", issues) + self.assertIn("ELBSecurityPolicy-2011-08 lacks server order cipher preference.", issues) + self.assertIn("ELBSecurityPolicy-2011-08 contains RC4 ciphers (RC4-SHA) that have been removed in newer policies.", issues) + self.assertIn("ELBSecurityPolicy-2011-08 contains a weaker cipher (DES-CBC3-SHA) " + "for backwards compatibility with Windows XP systems. Vulnerable to SWEET32 CVE-2016-2183.", issues) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2014-01', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 3) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2014-10', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 2) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2015-02', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 2) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2015-03', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 2) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2015-05', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 1) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-2016-08', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 0) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-TLS-1-1-2017-01', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 0) + + item.audit_issues = list() + auditor._process_reference_policy('ELBSecurityPolicy-TLS-1-2-2017-01', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 0) + + item.audit_issues = list() + auditor._process_reference_policy('OTHER_REFERENCE_POLICY', 'MyCustomPolicy', '443', item) + self.assertEqual(len(item.audit_issues), 1) + self.assertEqual(item.audit_issues[0].issue, 'Unknown reference policy.') + + def test_process_custom_listener_policy(self): + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + # We'll just modify it and pretend it's a custom policy + policy = dict(INTERNET_ELB['PolicyDescriptions']['ELBSecurityPolicy-2016-08']) + + auditor._process_custom_listener_policy('ELBSecurityPolicy-2016-08', policy, '443', item) + self.assertEqual(len(item.audit_issues), 1) + + item.audit_issues = list() + policy['protocols']['sslv2'] = True + auditor._process_custom_listener_policy('ELBSecurityPolicy-2016-08', policy, '443', item) + self.assertEqual(len(item.audit_issues), 2) + + item.audit_issues = list() + policy['server_defined_cipher_order'] = False + auditor._process_custom_listener_policy('ELBSecurityPolicy-2016-08', policy, '443', item) + self.assertEqual(len(item.audit_issues), 3) + + # simulate export grade + item.audit_issues = list() + policy['supported_ciphers'].append('EXP-RC4-MD5') + auditor._process_custom_listener_policy('ELBSecurityPolicy-2016-08', policy, '443', item) + self.assertEqual(len(item.audit_issues), 4) + + # simulate deprecated cipher + item.audit_issues = list() + policy['supported_ciphers'].append('RC2-CBC-MD5') + auditor._process_custom_listener_policy('ELBSecurityPolicy-2016-08', policy, '443', item) + self.assertEqual(len(item.audit_issues), 5) + + # simulate not-recommended cipher + item.audit_issues = list() + policy['supported_ciphers'].append('CAMELLIA128-SHA') + auditor._process_custom_listener_policy('ELBSecurityPolicy-2016-08', policy, '443', item) + self.assertEqual(len(item.audit_issues), 6) + + def test_check_listener_reference_policy(self): + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + auditor.check_listener_reference_policy(item) + self.assertEqual(len(item.audit_issues), 0) + + def test_check_logging(self): + from security_monkey.auditors.elb import ELBAuditor + auditor = ELBAuditor(accounts=["012345678910"]) + + from security_monkey.cloudaux_watcher import CloudAuxChangeItem + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + auditor.check_logging(item) + self.assertEqual(len(item.audit_issues), 0) + + elb = dict(INTERNET_ELB) + elb['Attributes']['AccessLog']['Enabled'] = False + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + auditor.check_logging(item) + self.assertEqual(len(item.audit_issues), 1) + self.assertEqual(item.audit_issues[0].issue, 'ELB is not configured for logging.') + + del elb['Attributes']['AccessLog'] + item = CloudAuxChangeItem(index='elb', account='TEST_ACCOUNT', name='MyELB', + arn="arn:aws:elasticloadbalancing:us-east-1:012345678910:loadbalancer/MyELB", config=INTERNET_ELB) + + auditor.check_logging(item) + self.assertEqual(len(item.audit_issues), 1) + self.assertEqual(item.audit_issues[0].issue, 'ELB is not configured for logging.') \ No newline at end of file diff --git a/security_monkey/tests/auditors/test_s3.py b/security_monkey/tests/auditors/test_s3.py index 0720c50a8..faa15a2ca 100644 --- a/security_monkey/tests/auditors/test_s3.py +++ b/security_monkey/tests/auditors/test_s3.py @@ -26,7 +26,7 @@ from security_monkey.tests import SecurityMonkeyTestCase from security_monkey import db -from security_monkey.watchers.s3 import S3Item +from security_monkey.cloudaux_watcher import CloudAuxChangeItem # With same account ownership: CONFIG_ONE = json.loads(b"""{ @@ -178,10 +178,10 @@ class S3AuditorTestCase(SecurityMonkeyTestCase): def pre_test_setup(self): self.s3_items = [ - S3Item(region="us-east-1", account="TEST_ACCOUNT", name="bucket1", config=CONFIG_ONE), - S3Item(region="us-east-1", account="TEST_ACCOUNT", name="bucket2", config=CONFIG_TWO), - S3Item(region="us-east-1", account="TEST_ACCOUNT2", name="bucket3", config=CONFIG_THREE), - S3Item(region="us-east-1", account="TEST_ACCOUNT3", name="bucket4", config=CONFIG_FOUR) + CloudAuxChangeItem(region="us-east-1", account="TEST_ACCOUNT", name="bucket1", config=CONFIG_ONE), + CloudAuxChangeItem(region="us-east-1", account="TEST_ACCOUNT", name="bucket2", config=CONFIG_TWO), + CloudAuxChangeItem(region="us-east-1", account="TEST_ACCOUNT2", name="bucket3", config=CONFIG_THREE), + CloudAuxChangeItem(region="us-east-1", account="TEST_ACCOUNT3", name="bucket4", config=CONFIG_FOUR) ] account_type_result = AccountType(name='AWS') diff --git a/security_monkey/tests/core/test_datastore_utils.py b/security_monkey/tests/core/test_datastore_utils.py index 1032622f4..be35c4805 100644 --- a/security_monkey/tests/core/test_datastore_utils.py +++ b/security_monkey/tests/core/test_datastore_utils.py @@ -213,7 +213,7 @@ def test_detect_change(self): complete_hash, durable_hash = hash_item(sti.config, []) # Item does not exist in the DB yet: - assert (True, 'durable', None) == detect_change(sti, self.account, self.technology, complete_hash, + assert (True, 'durable', None, 'created') == detect_change(sti, self.account, self.technology, complete_hash, durable_hash) # Add the item to the DB: @@ -221,7 +221,7 @@ def test_detect_change(self): db.session.commit() # Durable change (nothing hashed in DB yet) - assert (True, 'durable', item) == detect_change(sti, self.account, self.technology, complete_hash, + assert (True, 'durable', item, 'changed') == detect_change(sti, self.account, self.technology, complete_hash, durable_hash) # No change: @@ -230,7 +230,7 @@ def test_detect_change(self): db.session.add(item) db.session.commit() - assert (False, None, item) == detect_change(sti, self.account, self.technology, complete_hash, + assert (False, None, item, None) == detect_change(sti, self.account, self.technology, complete_hash, durable_hash) # Ephemeral change: @@ -238,7 +238,7 @@ def test_detect_change(self): mod_conf["IGNORE_ME"] = "I am ephemeral!" complete_hash, durable_hash = hash_item(mod_conf, ["IGNORE_ME"]) - assert (True, 'ephemeral', item) == detect_change(sti, self.account, self.technology, complete_hash, + assert (True, 'ephemeral', item, None) == detect_change(sti, self.account, self.technology, complete_hash, durable_hash) def test_persist_item(self): @@ -328,13 +328,9 @@ def test_inactivate_old_revisions(self): assert not item_revision.active - # Check that there are no issues associated with this item: - assert len(ItemAudit.query.filter(ItemAudit.item_id == item_revision.item_id).all()) == 0 - # Check that the SomeRole0 is still OK: item_revision = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)).filter( - Item.arn == "arn:aws:iam::012345678910:role/SomeRole0".format(x), - ).one() + Item.arn == "arn:aws:iam::012345678910:role/SomeRole0").one() assert len(ItemAudit.query.filter(ItemAudit.item_id == item_revision.item_id).all()) == 2 diff --git a/security_monkey/tests/core/test_monitors.py b/security_monkey/tests/core/test_monitors.py index 2a3d7ee50..25da5fe57 100644 --- a/security_monkey/tests/core/test_monitors.py +++ b/security_monkey/tests/core/test_monitors.py @@ -107,6 +107,7 @@ def pre_test_setup(self): account_result = Account( name='TEST_ACCOUNT', identifier='012345678910', + third_party=False, active=True, account_type_id=account_type.id ) db.session.add(account_result) diff --git a/security_monkey/tests/core/test_reporter.py b/security_monkey/tests/core/test_reporter.py index 7a879f7df..b2437c4dd 100644 --- a/security_monkey/tests/core/test_reporter.py +++ b/security_monkey/tests/core/test_reporter.py @@ -345,7 +345,9 @@ def test_report_batch_changes(self): db.session.add(account_type_result) db.session.commit() - test_account = Account(name="TEST_ACCOUNT") + test_account = Account(identifier="012345678910", name="TEST_ACCOUNT", + account_type_id=account_type_result.id, notes="TEST_ACCOUNT1", + third_party=False, active=True) watcher = IAMRole(accounts=[test_account.name]) db.session.commit() @@ -377,12 +379,12 @@ def test_report_batch_changes(self): original_slurp = watcher.slurp def mock_slurp_list(): - exception_map = original_slurp_list() + items, exception_map = original_slurp_list() for item in watcher.total_list: item["Arn"] = "arn:aws:iam::012345678910:role/{}".format(item["RoleName"]) - return exception_map + return items, exception_map def mock_slurp(): batched_items, exception_map = original_slurp() diff --git a/security_monkey/tests/core/test_scheduler.py b/security_monkey/tests/core/test_scheduler.py index 4d813d284..7bbe8cca8 100644 --- a/security_monkey/tests/core/test_scheduler.py +++ b/security_monkey/tests/core/test_scheduler.py @@ -276,12 +276,12 @@ def test_find_batch_changes(self): original_slurp = watcher.slurp def mock_slurp_list(): - exception_map = original_slurp_list() + items, exception_map = original_slurp_list() for item in watcher.total_list: item["Arn"] = "arn:aws:iam::012345678910:role/{}".format(item["RoleName"]) - return exception_map + return items, exception_map def mock_slurp(): batched_items, exception_map = original_slurp() @@ -320,8 +320,8 @@ def mock_slurp(): # Check that nothing new was added: assert len(Item.query.all()) == 11 - # There should be 2 less issues and 2 more revisions: - assert len(ItemAudit.query.all()) == 9 + # There should be the same number of issues and 2 more revisions: + assert len(ItemAudit.query.all()) == 11 assert len(ItemRevision.query.all()) == 13 # Check that the deleted roles show as being inactive: @@ -339,10 +339,10 @@ def mock_slurp_list_with_exception(): import security_monkey.watchers.iam.iam_role security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0 - exception_map = original_slurp_list() + items, exception_map = original_slurp_list() assert len(exception_map) > 0 - return exception_map + return items, exception_map watcher.slurp_list = mock_slurp_list_with_exception watcher.current_account = None # Need to reset the watcher @@ -354,7 +354,8 @@ def mock_slurp_list_with_exception(): def test_audit_specific_changes(self): from security_monkey.scheduler import _audit_specific_changes from security_monkey.monitors import Monitor - from security_monkey.watchers.iam.iam_role import IAMRole, IAMRoleItem + from security_monkey.watchers.iam.iam_role import IAMRole + from security_monkey.cloudaux_watcher import CloudAuxChangeItem from security_monkey.auditors.iam.iam_role import IAMRoleAuditor # Set up the monitor: @@ -376,7 +377,7 @@ def test_audit_specific_changes(self): role_policy = dict(ROLE_CONF) role_policy["Arn"] = "arn:aws:iam::012345678910:role/roleNumber{}".format(x) role_policy["RoleName"] = "roleNumber{}".format(x) - role = IAMRoleItem.from_slurp(role_policy, account_name=test_account.name) + role = CloudAuxChangeItem.from_item(name=role_policy['RoleName'], item=role_policy, override_region='universal', account_name=test_account.name, index='iamrole') items.append(role) audit_items = watcher.find_changes_batch(items, {}) diff --git a/security_monkey/tests/core/test_watcher.py b/security_monkey/tests/core/test_watcher.py index 35fd0a303..4c5c34c9e 100644 --- a/security_monkey/tests/core/test_watcher.py +++ b/security_monkey/tests/core/test_watcher.py @@ -109,7 +109,7 @@ def test_from_items(self): issue.justification = 'test justification' old_item_w_issues = ChangeItem(index='testtech', region='us-west-2', account='testaccount', - new_config=CONFIG_1, active=True, audit_issues=[issue]) + new_config=CONFIG_1, active=True, audit_issues=[issue]) old_item_wo_issues = ChangeItem(index='testtech', region='us-west-2', account='testaccount', new_config=CONFIG_1, active=True) new_item = ChangeItem(index='testtech', region='us-west-2', account='testaccount', new_config=CONFIG_2, @@ -127,6 +127,7 @@ def setup_batch_db(self): db.session.commit() self.account = Account(identifier="012345678910", name="testing", + active=True, third_party=False, account_type_id=account_type_result.id) self.technology = Technology(name="iamrole") @@ -173,6 +174,7 @@ def test_no_change_items(self): ) ] + self._setup_account() watcher = Watcher(accounts=['test_account']) watcher.find_modified(previous, current) @@ -218,6 +220,7 @@ def test_changed_item(self): ) ] + self._setup_account() watcher = Watcher(accounts=['test_account']) watcher.find_modified(previous, current) @@ -265,6 +268,7 @@ def test_ephemeral_change(self): ) ] + self._setup_account() watcher = Watcher(accounts=['test_account']) watcher.honor_ephemerals = True watcher.ephemeral_paths = ['test_ephemeral'] @@ -367,6 +371,7 @@ def _setup_account(self): db.session.commit() account = Account(identifier="012345678910", name="test_account", + third_party=False, active=True, account_type_id=account_type_result.id) db.session.add(account) @@ -396,9 +401,19 @@ def test_find_changes_batch(self): items.append(SomeTestItem().from_slurp(mod_conf, account_name=self.account.name)) assert len(watcher.find_changes(items)) == 5 + assert len(watcher.deleted_items) == 0 + assert len(watcher.changed_items) == 0 + assert len(watcher.created_items) == 5 + + watcher_2 = IAMRole(accounts=[self.account.name]) + watcher_2.current_account = (self.account, 0) + watcher_2.technology = self.technology # Try again -- audit_items should be 0 since nothing was changed: - assert len(watcher.find_changes(items)) == 0 + assert len(watcher_2.find_changes(items)) == 0 + assert len(watcher_2.deleted_items) == 0 + assert len(watcher_2.changed_items) == 0 + assert len(watcher_2.created_items) == 0 def test_find_deleted_batch(self): """ @@ -430,6 +445,7 @@ def test_find_deleted_batch(self): # Check for deleted items: watcher.find_deleted_batch({}) + assert len(watcher.deleted_items) == 0 # Check that nothing was deleted: for x in range(0, 5): @@ -456,6 +472,7 @@ def test_find_deleted_batch(self): # Check for deleted items again: watcher.find_deleted_batch({}) + assert len(watcher.deleted_items) == 2 # Check that the last two items were deleted: for arn in removed_arns: @@ -464,7 +481,6 @@ def test_find_deleted_batch(self): ).one() assert not item_revision.active - assert len(ItemAudit.query.filter(ItemAudit.item_id == item_revision.item_id).all()) == 0 # Check that the current ones weren't deleted: for current_item in watcher.total_list: @@ -474,5 +490,3 @@ def test_find_deleted_batch(self): assert item_revision.active assert len(ItemAudit.query.filter(ItemAudit.item_id == item_revision.item_id).all()) == 2 - - assert len(ItemAudit.query.all()) == len(watcher.total_list) * 2 diff --git a/security_monkey/tests/watchers/test_iam_role.py b/security_monkey/tests/watchers/test_iam_role.py index e5f2880e8..47955fbf0 100644 --- a/security_monkey/tests/watchers/test_iam_role.py +++ b/security_monkey/tests/watchers/test_iam_role.py @@ -35,6 +35,7 @@ def pre_test_setup(self): db.session.commit() self.account = Account(identifier="012345678910", name="testing", + third_party=False, active=True, account_type_id=account_type_result.id) self.technology = Technology(name="iamrole") @@ -81,7 +82,7 @@ def test_slurp_list(self): watcher = IAMRole(accounts=[self.account.name]) - exceptions = watcher.slurp_list() + _, exceptions = watcher.slurp_list() assert len(exceptions) == 0 assert len(watcher.total_list) == self.total_roles @@ -93,9 +94,9 @@ def test_empty_slurp_list(self): mock_sts().start() watcher = IAMRole(accounts=[self.account.name]) - watcher.list_roles = lambda **kwargs: [] + watcher.list_method = lambda **kwargs: [] - exceptions = watcher.slurp_list() + _, exceptions = watcher.slurp_list() assert len(exceptions) == 0 assert len(watcher.total_list) == 0 assert watcher.done_slurping @@ -110,10 +111,9 @@ def test_slurp_list_exceptions(self): def raise_exception(): raise Exception("LOL, HAY!") - import security_monkey.watchers.iam.iam_role - security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: raise_exception() + watcher.list_method = lambda **kwargs: raise_exception() - exceptions = watcher.slurp_list() + _, exceptions = watcher.slurp_list() assert len(exceptions) == 1 assert len(ExceptionLogs.query.all()) == 1 @@ -153,8 +153,7 @@ def test_slurp_items_with_exceptions(self): def raise_exception(): raise Exception("LOL, HAY!") - import security_monkey.watchers.iam.iam_role - security_monkey.watchers.iam.iam_role.get_role = lambda **kwargs: raise_exception() + watcher.get_method = lambda *args, **kwargs: raise_exception() items, exceptions = watcher.slurp() assert len(exceptions) == self.total_roles diff --git a/security_monkey/tests/watchers/test_s3.py b/security_monkey/tests/watchers/test_s3.py index 44f72e0c1..aa801e82b 100644 --- a/security_monkey/tests/watchers/test_s3.py +++ b/security_monkey/tests/watchers/test_s3.py @@ -35,6 +35,7 @@ def pre_test_setup(self): db.session.commit() self.account = Account(identifier="012345678910", name="testing", + active=True, third_party=False, account_type_id=account_type_result.id) self.technology = Technology(name="s3") self.item = Item(region="us-west-2", name="somebucket", diff --git a/security_monkey/watcher.py b/security_monkey/watcher.py index 9a3fe5178..1b2a904e2 100644 --- a/security_monkey/watcher.py +++ b/security_monkey/watcher.py @@ -27,12 +27,13 @@ from dpath.exceptions import PathNotFound watcher_registry = {} +abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher']) class WatcherType(type): def __init__(cls, name, bases, attrs): super(WatcherType, cls).__init__(name, bases, attrs) - if cls.__name__ != 'Watcher' and cls.index: + if cls.__name__ not in abstract_classes and cls.index: app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__)) watcher_registry[cls.index] = cls @@ -55,9 +56,12 @@ def __init__(self, accounts=None, debug=False): self.datastore = datastore.Datastore() if not accounts: accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).all() - self.accounts = [account.name for account in accounts] else: - self.accounts = accounts + accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).filter(Account.name.in_(accounts)).all() + if not accounts: + raise ValueError('Watcher needs a valid account') + self.accounts = [account.name for account in accounts] + self.account_identifiers = [account.identifier for account in accounts] self.debug = debug self.created_items = [] self.deleted_items = [] @@ -377,29 +381,49 @@ def find_changes_batch(self, items, exception_map): complete_hash, durable_hash = hash_item(item.config, self.ephemeral_paths) # Detect if a change occurred: - is_change, change_type, db_item = detect_change(item, self.current_account[0], self.technology, - complete_hash, durable_hash) + is_change, change_type, db_item, created_changed = detect_change( + item, self.current_account[0], self.technology, complete_hash, durable_hash) - # As Officer Barbrady says: "Move along... Nothing to see here..." if not is_change: continue - # Now call out to persist item: is_durable = (change_type == "durable") - persist_item(item, db_item, self.technology, self.current_account[0], complete_hash, - durable_hash, is_durable) - if is_durable: durable_items.append(item) + if created_changed == 'created': + self.created_items.append(ChangeItem.from_items(old_item=None, new_item=item)) + + if created_changed == 'changed': + db_item.audit_issues = db_item.issues + db_item.config = db_item.revisions.first().config + self.changed_items.append(ChangeItem.from_items(old_item=db_item, new_item=item)) + + persist_item(item, db_item, self.technology, self.current_account[0], complete_hash, + durable_hash, is_durable) + return durable_items def find_deleted_batch(self, exception_map): - arns = [item["Arn"] for item in self.total_list] - from datastore_utils import inactivate_old_revisions - return inactivate_old_revisions(self, arns, self.current_account[0], self.technology) + existing_arns = [item["Arn"] for item in self.total_list] + deleted_items = inactivate_old_revisions(self, existing_arns, self.current_account[0], self.technology) + + for item in deleted_items: + # An inactive revision has already been commited to the DB. + # So here, we need to pull the last two revisions to build out our + # ChangeItem. + recent_revisions=item.revisions.limit(2).all() + old_config=recent_revisions[1].config + new_config=recent_revisions[0].config + change_item = ChangeItem( + index=item.technology.name, region=item.region, + account=item.account.name, name=item.name, arn=item.arn, + old_config=old_config, new_config=new_config, active=False, + audit_issues=item.issues) + self.deleted_items.append(change_item) + def read_previous_items(self): """ diff --git a/security_monkey/watchers/elb.py b/security_monkey/watchers/elb.py index df82a5c4d..d463b04f6 100644 --- a/security_monkey/watchers/elb.py +++ b/security_monkey/watchers/elb.py @@ -1,257 +1,28 @@ -# Copyright 2014 Netflix, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -.. module: security_monkey.watchers.elb - :platform: Unix - -.. version:: $$VERSION$$ -.. moduleauthor:: Patrick Kelley @monkeysecurity - -""" -from security_monkey.watcher import Watcher -from security_monkey.watcher import ChangeItem -from security_monkey.constants import TROUBLE_REGIONS -from security_monkey.exceptions import BotoConnectionIssue -from security_monkey.datastore import Account +from security_monkey.cloudaux_watcher import CloudAuxWatcher +from cloudaux.aws.elb import describe_load_balancers +from cloudaux.orchestration.aws.elb import get_load_balancer from security_monkey import app -from boto.ec2.elb import regions - - -def parse_policy(policy): - ret = {} - ret['name'] = policy['name'] - ret['type'] = policy['type'] - attrs = policy['Attributes'] - - if policy['type'] != 'SSLNegotiationPolicyType': - return ret - - ret['sslv2'] = bool(attrs.get('Protocol-SSLv2')) - ret['sslv3'] = bool(attrs.get('Protocol-SSLv3')) - ret['tlsv1'] = bool(attrs.get('Protocol-TLSv1')) - ret['tlsv1_1'] = bool(attrs.get('Protocol-TLSv1.1')) - ret['tlsv1_2'] = bool(attrs.get('Protocol-TLSv1.2')) - ret['server_defined_cipher_order'] = bool(attrs.get('Server-Defined-Cipher-Order')) - ret['reference_security_policy'] = attrs.get('Reference-Security-Policy', None) - - non_ciphers = [ - 'Server-Defined-Cipher-Order', - 'Protocol-SSLv2', - 'Protocol-SSLv3', - 'Protocol-TLSv1', - 'Protocol-TLSv1.1', - 'Protocol-TLSv1.2', - 'Reference-Security-Policy' - ] - - ciphers = [] - for cipher in attrs: - if attrs[cipher] and cipher not in non_ciphers: - ciphers.append(cipher) - - ciphers.sort() - ret['supported_ciphers'] = ciphers - - return ret - -class ELB(Watcher): +class ELB(CloudAuxWatcher): index = 'elb' i_am_singular = 'ELB' i_am_plural = 'ELBs' - - def __init__(self, accounts=None, debug=False): - super(ELB, self).__init__(accounts=accounts, debug=debug) - - def _setup_botocore(self, account): - from security_monkey.common.sts_connect import connect - self.botocore_session = connect(account, 'botocore') - - def _get_listener_policies(self, operation, elb): - response_data = self.wrap_aws_rate_limited_call(operation, LoadBalancerName=elb.name) - policies = {} - for policy in response_data.get('PolicyDescriptions', []): - p = {"name": policy['PolicyName'], "type": policy['PolicyTypeName'], "Attributes": {}} - for attribute in policy['PolicyAttributeDescriptions']: - if attribute['AttributeValue'] == "true": - p['Attributes'][attribute['AttributeName']] = True - elif attribute['AttributeValue'] == "false": - p['Attributes'][attribute['AttributeName']] = False - else: - p['Attributes'][attribute['AttributeName']] = attribute['AttributeValue'] - - # This next bit may overwrite anything you did in the above for-loop: - if "Reference-Security-Policy" in p['Attributes']: - p['reference_security_policy'] = p['Attributes']['Reference-Security-Policy'] - del p['Attributes'] - else: - p = parse_policy(p) - - policies[policy['PolicyName']] = p - - return policies - - def slurp(self): - """ - :returns: item_list - list of ELB's. - :returns: exception_map - A dict where the keys are a tuple containing the - location of the exception and the value is the actual exception - - """ - self.prep_for_slurp() - from security_monkey.common.sts_connect import connect - item_list = [] - exception_map = {} - for account in self.accounts: - account_db = Account.query.filter(Account.name == account).first() - account_number = account_db.identifier - - try: - self._setup_botocore(account) - except Exception as e: - self.slurp_exception((self.index, account), e, exception_map) - continue - - for region in regions(): - app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) - try: - elb_conn = connect(account, 'ec2.elb', region=region.name) - - botocore_client = self.botocore_session.create_client('elb', region_name=region.name) - botocore_operation = botocore_client.describe_load_balancer_policies - - all_elbs = [] - marker = None - - while True: - response = self.wrap_aws_rate_limited_call( - elb_conn.get_all_load_balancers, - marker=marker - ) - - # build our elb list - all_elbs.extend(response) - - # ensure that we get every elb - if response.next_marker: - marker = response.next_marker - else: - break - - except Exception as e: - if region.name not in TROUBLE_REGIONS: - exc = BotoConnectionIssue(str(e), self.index, account, region.name) - self.slurp_exception((self.index, account, region.name), exc, exception_map, - source="{}-watcher".format(self.index)) - continue - - app.logger.debug("Found {} {}".format(len(all_elbs), self.i_am_plural)) - for elb in all_elbs: - - if self.check_ignore_list(elb.name): - continue - - try: - elb_map = {} - elb_map['availability_zones'] = list(elb.availability_zones) - elb_map['canonical_hosted_zone_name'] = elb.canonical_hosted_zone_name - elb_map['canonical_hosted_zone_name_id'] = elb.canonical_hosted_zone_name_id - elb_map['dns_name'] = elb.dns_name - elb_map['health_check'] = {'target': elb.health_check.target, 'interval': elb.health_check.interval} - elb_map['is_cross_zone_load_balancing'] = self.wrap_aws_rate_limited_call( - elb.is_cross_zone_load_balancing - ) - elb_map['scheme'] = elb.scheme - elb_map['security_groups'] = list(elb.security_groups) - elb_map['source_security_group'] = elb.source_security_group.name - elb_map['subnets'] = list(elb.subnets) - elb_map['vpc_id'] = elb.vpc_id - elb_map['is_logging'] = self.wrap_aws_rate_limited_call( - lambda: elb.get_attributes().access_log.enabled - ) - - backends = [] - for be in elb.backends: - backend = {} - backend['instance_port'] = be.instance_port - policies = [] - for bepol in be.policies: - policies.append(bepol.policy_name) - backend['policies'] = policies - backends.append(backend) - elb_map['backends'] = backends - - elb_policies = self._get_listener_policies(botocore_operation, elb) - listeners = [] - for li in elb.listeners: - listener = { - 'load_balancer_port': li.load_balancer_port, - 'instance_port': li.instance_port, - 'protocol': li.protocol, - 'instance_protocol': li.instance_protocol, - 'ssl_certificate_id': li.ssl_certificate_id, - 'policies': [elb_policies.get(policy_name, {"name": policy_name}) for policy_name in li.policy_names] - } - listeners.append(listener) - elb_map['listeners'] = listeners - - policies = {} - app_cookie_stickiness_policies = [] - for policy in elb.policies.app_cookie_stickiness_policies: - app_cookie_stickiness_policy = {} - app_cookie_stickiness_policy['policy_name'] = policy.policy_name - app_cookie_stickiness_policy['cookie_name'] = policy.cookie_name - app_cookie_stickiness_policies.append(app_cookie_stickiness_policy) - policies['app_cookie_stickiness_policies'] = app_cookie_stickiness_policies - - lb_cookie_stickiness_policies = [] - for policy in elb.policies.lb_cookie_stickiness_policies: - lb_cookie_stickiness_policy = {} - lb_cookie_stickiness_policy['policy_name'] = policy.policy_name - lb_cookie_stickiness_policy['cookie_expiration_period'] = policy.cookie_expiration_period - lb_cookie_stickiness_policies.append(lb_cookie_stickiness_policy) - policies['lb_cookie_stickiness_policies'] = lb_cookie_stickiness_policies - - policies['other_policies'] = [] - for opol in elb.policies.other_policies: - policies['other_policies'].append(opol.policy_name) - elb_map['policies'] = policies - - arn = 'arn:aws:elasticloadbalancing:{region}:{account_number}:loadbalancer/{name}'.format( - region=region.name, - account_number=account_number, - name=elb.name) - - elb_map['arn'] = arn - - item = ELBItem(region=region.name, account=account, name=elb.name, arn=arn, config=elb_map) - item_list.append(item) - except Exception as e: - self.slurp_exception((self.index, account, region.name, elb.name), e, exception_map, - source="{}-watcher".format(self.index)) - continue - - return item_list, exception_map - - -class ELBItem(ChangeItem): - def __init__(self, region=None, account=None, name=None, arn=None, config={}): - super(ELBItem, self).__init__( - index=ELB.index, - region=region, - account=account, - name=name, - arn=arn, - new_config=config) + honor_ephemerals = False + ephemeral_paths = list() + service_name = 'elb' + detail = app.config.get('SECURITYGROUP_INSTANCE_DETAIL', 'FULL') + + def get_name_from_list_output(self, item): + return item['LoadBalancerName'] + + def list_method(self, **kwargs): + return describe_load_balancers(**kwargs) + + def get_method(self, item, **kwargs): + result = get_load_balancer(item, **kwargs) + if self.detail == 'NONE' or self.detail == None: + result.pop('Instances', None) + elif self.detail == 'SUMMARY': + result['Instances'] = '{len} Instance(s)'.format(len=len(result.get('Instances', []))) + return result diff --git a/security_monkey/watchers/elbv2.py b/security_monkey/watchers/elbv2.py new file mode 100644 index 000000000..7c156d7d7 --- /dev/null +++ b/security_monkey/watchers/elbv2.py @@ -0,0 +1,21 @@ +from security_monkey.cloudaux_watcher import CloudAuxWatcher +from cloudaux.aws.elbv2 import describe_load_balancers +from cloudaux.orchestration.aws.elbv2 import get_elbv2 + + +class ELBv2(CloudAuxWatcher): + index = 'alb' + i_am_singular = 'ALB' + i_am_plural = 'ALBs' + honor_ephemerals = False + ephemeral_paths = list() + service_name = 'elbv2' + + def get_name_from_list_output(self, item): + return item['LoadBalancerName'] + + def list_method(self, **kwargs): + return describe_load_balancers(**kwargs) + + def get_method(self, item, **kwargs): + return get_elbv2(item, **kwargs) diff --git a/security_monkey/watchers/gcp/gce/firewall.py b/security_monkey/watchers/gcp/gce/firewall.py index 3e4d3870c..afd4c4806 100644 --- a/security_monkey/watchers/gcp/gce/firewall.py +++ b/security_monkey/watchers/gcp/gce/firewall.py @@ -24,7 +24,6 @@ from cloudaux.gcp.decorators import iter_project from cloudaux.gcp.gce.firewall import list_firewall_rules -from cloudaux.orchestration import modify class GCEFirewallRule(Watcher): @@ -58,14 +57,14 @@ def slurp_items(**kwargs): for rule in rules: resource_id = gcp_resource_id_builder( - 'compute.firewall.get', rule['name']) + kwargs['project'], 'compute.firewall.get', rule['name']) item_list.append( GCEFirewallRuleItem( region='global', account=kwargs['project'], name=rule['name'], arn=resource_id, - config=modify(rule, format='camelized'))) + config=modify(rule, output='camelized'))) return item_list, kwargs.get('exception_map', {}) return slurp_items() diff --git a/security_monkey/watchers/gcp/gce/network.py b/security_monkey/watchers/gcp/gce/network.py index c71f8ba43..c76ce50e1 100644 --- a/security_monkey/watchers/gcp/gce/network.py +++ b/security_monkey/watchers/gcp/gce/network.py @@ -60,7 +60,7 @@ def slurp_items(**kwargs): for network in networks: resource_id = gcp_resource_id_builder( - 'compute.network.get', network['name']) + kwargs['project'], 'compute.network.get', network['name']) net_complete = get_network_and_subnetworks( network['name'], **kwargs) item_list.append( diff --git a/security_monkey/watchers/gcp/gcs/bucket.py b/security_monkey/watchers/gcp/gcs/bucket.py index df85bc0bc..ae5d7c331 100644 --- a/security_monkey/watchers/gcp/gcs/bucket.py +++ b/security_monkey/watchers/gcp/gcs/bucket.py @@ -60,7 +60,7 @@ def slurp_items(**kwargs): for bucket in buckets: resource_id = gcp_resource_id_builder( - 'storage.bucket.get', bucket['name']) + kwargs['project'], 'storage.bucket.get', bucket['name']) b = get_bucket( bucket_name=bucket['name'], **kwargs) item_list.append( diff --git a/security_monkey/watchers/gcp/iam/serviceaccount.py b/security_monkey/watchers/gcp/iam/serviceaccount.py index 9b83704ac..14ccd5de4 100644 --- a/security_monkey/watchers/gcp/iam/serviceaccount.py +++ b/security_monkey/watchers/gcp/iam/serviceaccount.py @@ -60,7 +60,7 @@ def slurp_items(**kwargs): for service_account in service_accounts: resource_id = gcp_resource_id_builder( - 'projects.serviceaccounts.get', service_account['name']) + kwargs['project'], 'projects.serviceaccounts.get', service_account['name']) sa = get_serviceaccount_complete( service_account=service_account['name'], **kwargs) @@ -72,7 +72,7 @@ def slurp_items(**kwargs): IAMServiceAccountItem( region='global', account=sa['ProjectId'], - name=sa['DisplayName'], + name=sa['Email'], arn=resource_id, config={ 'policy': sa.get('Policy', None), diff --git a/security_monkey/watchers/iam/iam_role.py b/security_monkey/watchers/iam/iam_role.py index 2515752c5..f240fb78b 100644 --- a/security_monkey/watchers/iam/iam_role.py +++ b/security_monkey/watchers/iam/iam_role.py @@ -1,148 +1,27 @@ -# Copyright 2014 Netflix, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -.. module: security_monkey.watchers.iam.iam_role - :platform: Unix - -.. version:: $$VERSION$$ -.. moduleauthor:: Mike Grima -.. moduleauthor:: Patrick Kelley @monkeysecurity - -""" -from cloudaux.orchestration.aws.iam.role import get_role +from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher from cloudaux.aws.iam import list_roles -from cloudaux.decorators import iter_account_region - -from security_monkey.decorators import record_exception -from security_monkey.watcher import ChangeItem -from security_monkey.watcher import Watcher -from security_monkey import app +from cloudaux.orchestration.aws.iam.role import get_role -class IAMRole(Watcher): +class IAMRole(CloudAuxBatchedWatcher): index = 'iamrole' i_am_singular = 'IAM Role' i_am_plural = 'IAM Roles' + honor_ephemerals = False + ephemeral_paths = list() + override_region = 'universal' - def __init__(self, accounts=None, debug=False): - super(IAMRole, self).__init__(accounts=accounts, debug=debug) - - self.batched_size = 100 - self.done_slurping = False - self.next_role = 0 - - @record_exception(source="iamrole-watcher", pop_exception_fields=True) - def list_roles(self, **kwargs): - roles = list_roles(**kwargs["conn_dict"]) - return [role for role in roles if not self.check_ignore_list(role['RoleName'])] - - @record_exception(source="iamrole-watcher", pop_exception_fields=True) - def process_role(self, role, **kwargs): - app.logger.debug("Slurping {index} ({name}) from {account}/{region}".format( - index=self.i_am_singular, - name=role['RoleName'], - account=kwargs["conn_dict"]["account_number"], - region=kwargs["conn_dict"]["region"])) - - # Need to send a copy, since we don't want to alter the total list! - return get_role(dict(role), **kwargs["conn_dict"]) - - def slurp_list(self): - self.prep_for_batch_slurp() - exception_map = {} - - @iter_account_region("iam", accounts=[self.current_account[0].identifier], session_name="SecurityMonkey", - assume_role=self.current_account[0].getCustom("role_name") or 'SecurityMonkey', - regions=["us-east-1"], conn_type="dict") - def get_role_list(**kwargs): - app.logger.debug("Fetching the full list of {index} that need to be slurped from {account}" - "/{region}...".format(index=self.i_am_plural, - account=self.current_account[0].name, - region=kwargs["conn_dict"]["region"])) - roles = self.list_roles(index=self.index, exception_record_region="universal", - account_name=self.current_account[0].name, - exception_map=exception_map, - **kwargs) - - # Are there any roles? - if not roles: - self.done_slurping = True - roles = [] - - return roles - - for r in get_role_list(): - self.total_list.extend(r) - - return exception_map - - def slurp(self): - exception_map = {} - batched_items = [] - - @iter_account_region("iam", accounts=[self.current_account[0].identifier], session_name="SecurityMonkey", - assume_role=self.current_account[0].getCustom("role_name") or 'SecurityMonkey', - regions=["us-east-1"], conn_type="dict") - def slurp_items(**kwargs): - item_list = [] # Only one region, so just keeping in iter_account_region... - - # This sets the role counting index -- which will then be incremented as things progress... - role_counter = self.batch_counter * self.batched_size - while self.batched_size - len(item_list) > 0 and not self.done_slurping: - current_role = self.total_list[role_counter] - role = self.process_role(current_role, name=current_role["RoleName"], - index=self.index, exception_record_region="universal", - account_name=self.current_account[0].name, - exception_map=exception_map, - **kwargs) - if role: - item = IAMRoleItem.from_slurp(role, account_name=self.current_account[0].name, **kwargs) - item_list.append(item) - - # If an exception is encountered -- skip the role for now... - role_counter += 1 - - # Are we done yet? - if role_counter == len(self.total_list): - self.done_slurping = True - - self.batch_counter += 1 - - return item_list - - for r in slurp_items(): - batched_items.extend(r) - - return batched_items, exception_map - + def __init__(self, **kwargs): + super(IAMRole, self).__init__(**kwargs) -class IAMRoleItem(ChangeItem): - def __init__(self, account=None, name=None, arn=None, config=None): - config = config or {} + def _get_regions(self): + return ['us-east-1'] + + def get_name_from_list_output(self, item): + return item['RoleName'] - super(IAMRoleItem, self).__init__( - index=IAMRole.index, - region='universal', - account=account, - name=name, - arn=arn, - new_config=config) + def list_method(self, **kwargs): + return list_roles(**kwargs) - @classmethod - def from_slurp(cls, role, **kwargs): - return cls( - account=kwargs['account_name'], - name=role['RoleName'], - config=role, - arn=role['Arn']) + def get_method(self, item, **kwargs): + return get_role(dict(item), **kwargs) diff --git a/security_monkey/watchers/iam/iam_user.py b/security_monkey/watchers/iam/iam_user.py index 2c6a26b07..e4b2165c0 100644 --- a/security_monkey/watchers/iam/iam_user.py +++ b/security_monkey/watchers/iam/iam_user.py @@ -1,92 +1,31 @@ -# Copyright 2014 Netflix, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -.. module: security_monkey.watchers.iam.iam_user - :platform: Unix - -.. version:: $$VERSION$$ -.. moduleauthor:: Patrick Kelley @monkeysecurity - -""" -from cloudaux.orchestration.aws.iam.user import get_user +from security_monkey.cloudaux_watcher import CloudAuxWatcher from cloudaux.aws.iam import list_users -from security_monkey.decorators import record_exception, iter_account_region -from security_monkey.watcher import ChangeItem -from security_monkey.watcher import Watcher -from security_monkey import app +from cloudaux.orchestration.aws.iam.user import get_user -class IAMUser(Watcher): +class IAMUser(CloudAuxWatcher): index = 'iamuser' i_am_singular = 'IAM User' i_am_plural = 'IAM Users' - def __init__(self, accounts=None, debug=False): - super(IAMUser, self).__init__(accounts=accounts, debug=debug) + def __init__(self, *args, **kwargs): + super(IAMUser, self).__init__(*args, **kwargs) self.honor_ephemerals = True self.ephemeral_paths = [ "PasswordLastUsed", "AccessKeys$*$LastUsedDate", "AccessKeys$*$Region", - "AccessKeys$*$ServiceName" - ] - - @record_exception(source="iamuser-watcher", pop_exception_fields=True) - def list_users(self, **kwargs): - users = list_users(**kwargs) - return [user for user in users if not self.check_ignore_list(user['UserName'])] - - @record_exception(source="iamuser-watcher", pop_exception_fields=True) - def process_user(self, user, **kwargs): - app.logger.debug("Slurping {index} ({name}) from {account}".format( - index=self.i_am_singular, - name=user['UserName'], - account=kwargs['account_number'])) - return get_user(user, **kwargs) - - def slurp(self): - self.prep_for_slurp() - - @iter_account_region(index=self.index, accounts=self.accounts, exception_record_region='universal') - def slurp_items(**kwargs): - item_list = [] - users = self.list_users(**kwargs) - - for user in users: - user = self.process_user(user, name=user['UserName'], **kwargs) - if user: - item = IAMUserItem.from_slurp(user, **kwargs) - item_list.append(item) + "AccessKeys$*$ServiceName"] + self.override_region = 'universal' - return item_list, kwargs.get('exception_map', {}) - return slurp_items() + def get_name_from_list_output(self, item): + return item['UserName'] + def _get_regions(self): + return ['us-east-1'] -class IAMUserItem(ChangeItem): - def __init__(self, account=None, name=None, arn=None, config={}): - super(IAMUserItem, self).__init__( - index=IAMUser.index, - region='universal', - account=account, - name=name, - arn=arn, - new_config=config) + def list_method(self, **kwargs): + return list_users(**kwargs) - @classmethod - def from_slurp(cls, user, **kwargs): - return cls( - account=kwargs['account_name'], - name=user['UserName'], - config=user, - arn=user['Arn']) + def get_method(self, item, **kwargs): + return get_user(item, **kwargs) diff --git a/security_monkey/watchers/iam/managed_policy.py b/security_monkey/watchers/iam/managed_policy.py index 5d0434c9e..0e9b853d7 100644 --- a/security_monkey/watchers/iam/managed_policy.py +++ b/security_monkey/watchers/iam/managed_policy.py @@ -68,6 +68,18 @@ def slurp(self): if self.check_ignore_list(policy.arn): continue + # Skip retrieving attached entities for policies with zero attachments. + attached_users = [] + attached_roles = [] + attached_groups = [] + if policy.attachment_count > 0: + app.logger.debug("Finding attachments for policy %s" % policy.policy_name) + attached_users = [a.arn for a in policy.attached_users.all()] + attached_roles = [a.arn for a in policy.attached_roles.all()] + attached_groups = [a.arn for a in policy.attached_groups.all()] + else: + app.logger.debug("Skipping policy attachment retrieval for policy %s because it has no attachments" % policy.policy_name) + item_config = { 'name': policy.policy_name, 'arn': policy.arn, @@ -75,9 +87,9 @@ def slurp(self): 'update_date': str(policy.update_date), 'default_version_id': policy.default_version_id, 'attachment_count': policy.attachment_count, - 'attached_users': [a.arn for a in policy.attached_users.all()], - 'attached_groups': [a.arn for a in policy.attached_groups.all()], - 'attached_roles': [a.arn for a in policy.attached_roles.all()], + 'attached_users': attached_users, + 'attached_groups': attached_groups, + 'attached_roles': attached_roles, 'policy': policy.default_version.document } diff --git a/security_monkey/watchers/iam/saml_provider.py b/security_monkey/watchers/iam/saml_provider.py new file mode 100644 index 000000000..072ae84e2 --- /dev/null +++ b/security_monkey/watchers/iam/saml_provider.py @@ -0,0 +1,25 @@ +from security_monkey.cloudaux_watcher import CloudAuxWatcher +from cloudaux.aws.iam import list_saml_providers +from cloudaux.orchestration.aws.iam.saml_provider import get_saml_provider + + +class SAMLProvider(CloudAuxWatcher): + index = 'samlprovider' + i_am_singular = 'SAML Provider' + i_am_plural = 'SAML Providers' + honor_ephemerals = False + ephemeral_paths = list() + override_region = 'universal' + + def get_name_from_list_output(self, item): + # Extract the name from the ARN + return item['Arn'].split('/')[-1] + + def _get_regions(self): + return ['us-east-1'] + + def list_method(self, **kwargs): + return list_saml_providers(**kwargs) + + def get_method(self, item, **kwargs): + return get_saml_provider(item, **kwargs) diff --git a/security_monkey/watchers/s3.py b/security_monkey/watchers/s3.py index 115350033..82b132053 100644 --- a/security_monkey/watchers/s3.py +++ b/security_monkey/watchers/s3.py @@ -1,99 +1,34 @@ -# Copyright 2014 Netflix, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -.. module: security_monkey.watchers.s3 - :platform: Unix - -.. version:: $$VERSION$$ -.. moduleauthor:: Patrick Kelley @monkeysecurity - -""" -from cloudaux.orchestration.aws.s3 import get_bucket -from cloudaux.aws.s3 import list_buckets -from security_monkey.decorators import record_exception, iter_account_region +from security_monkey.cloudaux_watcher import CloudAuxWatcher from security_monkey.exceptions import SecurityMonkeyException -from security_monkey.watcher import ChangeItem -from security_monkey.watcher import Watcher -from security_monkey import app +from cloudaux.aws.s3 import list_buckets +from cloudaux.orchestration.aws.s3 import get_bucket -class S3(Watcher): +class S3(CloudAuxWatcher): index = 's3' i_am_singular = 'S3 Bucket' i_am_plural = 'S3 Buckets' - def __init__(self, accounts=None, debug=False): - super(S3, self).__init__(accounts=accounts, debug=debug) - - self.ephemeral_paths = ["GrantReferences"] + def __init__(self, *args, **kwargs): + super(S3, self).__init__(*args, **kwargs) self.honor_ephemerals = True + self.ephemeral_paths = ['GrantReferences'] + self.service_name = 's3' - @record_exception(source="s3-watcher", pop_exception_fields=True) - def list_buckets(self, **kwargs): - buckets = list_buckets(**kwargs) - return [bucket['Name'] for bucket in buckets['Buckets'] if not self.check_ignore_list(bucket['Name'])] + def list_method(self, **kwargs): + buckets = list_buckets(**kwargs)['Buckets'] + return [bucket['Name'] for bucket in buckets] - @record_exception(source="s3-watcher", pop_exception_fields=True) - def process_bucket(self, bucket_name, **kwargs): - app.logger.debug("Slurping {index} ({name}) from {account}".format( - index=self.i_am_singular, - name=bucket_name, - account=kwargs['account_number'])) - bucket = get_bucket(bucket_name, **kwargs) + def get_name_from_list_output(self, item): + return item - if bucket and bucket.get("Error"): - raise SecurityMonkeyException("S3 Bucket: {} fetching error: {}".format(bucket_name, bucket["Error"])) + def _get_regions(self): + return ['us-east-1'] - return bucket + def get_method(self, item_name, **kwargs): + bucket = get_bucket(item_name, **kwargs) - def slurp(self): - self.prep_for_slurp() - - @iter_account_region(index=self.index, accounts=self.accounts) - def slurp_items(**kwargs): - item_list = [] - bucket_names = self.list_buckets(**kwargs) - - for bucket_name in bucket_names: - bucket = self.process_bucket(bucket_name, name=bucket_name, **kwargs) - - if bucket: - if bucket.has_key('Error'): - app.logger.warn("Couldn't obtain ACL for S3 bucket {}. Error: {}".format(bucket_name, bucket['Error'])) - else: - item = S3Item.from_slurp(bucket_name, bucket, **kwargs) - item_list.append(item) - - return item_list, kwargs.get('exception_map', {}) - return slurp_items() - - -class S3Item(ChangeItem): - def __init__(self, account=None, region='us-east-1', name=None, arn=None, config={}): - super(S3Item, self).__init__( - index=S3.index, - region=region, - account=account, - name=name, - arn=arn, - new_config=config) + if bucket and bucket.get("Error"): + raise SecurityMonkeyException("S3 Bucket: {} fetching error: {}".format(item_name, bucket["Error"])) - @classmethod - def from_slurp(cls, bucket_name, bucket, **kwargs): - return cls( - account=kwargs['account_name'], - name=bucket_name, - region=bucket['Region'], - config=bucket, - arn=bucket['Arn']) + return bucket diff --git a/setup.py b/setup.py index 4efec3f09..da03d16cf 100644 --- a/setup.py +++ b/setup.py @@ -55,15 +55,14 @@ 'itsdangerous==0.23', 'psycopg2==2.6.2', 'bcrypt==3.1.2', - 'Sphinx==1.5.1', 'gunicorn==18.0', - 'cryptography==1.7.1', + 'cryptography>=1.8.1', 'boto3>=1.4.2', 'botocore>=1.4.81', 'dpath==1.3.2', 'pyyaml==3.11', 'jira==0.32', - 'cloudaux>=1.1.5', + 'cloudaux>=1.2.6', 'joblib>=0.9.4', 'pyjwt>=1.01' ], @@ -74,8 +73,7 @@ 'mixer==5.5.7', 'mock==1.0.1', 'moto==0.4.30', - 'freezegun>=0.3.7', - 'mixer==5.5.7' + 'freezegun>=0.3.7' ] }, entry_points={