Release 1.0

.gitignore

@@ -6,6 +6,7 @@
 # Packages
 *.egg
 *.egg-info
+.python-eggs
 dist
 build
 _build
@@ -61,3 +62,8 @@ secmonkey.env
 *.crt
 *.key
 postgres-data/
+docker-compose.override.yml
+.cache/
+dart/pubspec.lock
+celerybeat-schedule
+celerybeat.pid

.travis.yml

@@ -13,7 +13,6 @@ matrix:
       env:
       - UNIT_TEST_JOB=true
       - PIP_DOWNLOAD_CACHE=".pip_download_cache"
-      - SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config.py
 
       addons:
         postgresql: "9.4"
@@ -23,6 +22,12 @@ matrix:
       - psql -c "CREATE ROLE securitymonkeyuser LOGIN PASSWORD 'securitymonkeypassword';" -U postgres
       - psql -c "CREATE SCHEMA secmonkey GRANT Usage, Create ON SCHEMA secmonkey TO securitymonkeyuser;" -U postgres
       - psql -c "set timezone TO 'GMT';" -U postgres
+      - pip install pip --upgrade
+      - pip install setuptools --upgrade
+      - pip install google-compute-engine
+      - pip install openstacksdk
+      - pip install cloudaux\[gcp\]
+      - pip install cloudaux\[openstack\]
       - python setup.py develop
       - pip install .[tests]
       - pip install coveralls
@@ -43,26 +48,24 @@ matrix:
         - coverage run -a -m py.test security_monkey/tests/auditors || exit 1
         - coverage run -a -m py.test security_monkey/tests/watchers || exit 1
         - coverage run -a -m py.test security_monkey/tests/core || exit 1
+        - coverage run -a -m py.test security_monkey/tests/scheduling || exit 1
         - coverage run -a -m py.test security_monkey/tests/views || exit 1
         - coverage run -a -m py.test security_monkey/tests/interface || exit 1
         - coverage run -a -m py.test security_monkey/tests/utilities || exit 1
         - bandit -r -ll -ii -x security_monkey/tests .
-        - pylint -E -d E1101,E0611,F0401 --ignore=service.py,datastore.py,datastore_utils.py,watcher.py security_monkey
+        - pylint -E -d E1101,E0611,F0401 --ignore=service.py,datastore.py,datastore_utils.py,watcher.py,test_celery_scheduler.py security_monkey
 
       after_success:
         - coveralls
         - coverage report
     #####################################################
 
     ##### BUILD DOCKER CONTAINER JOB #####
-    - dist: trusty
-      sudo: required
-      language: python
-      python: "2.7"
+    - sudo: required
 
       env:
         - BUILD_DOCKER=True
-        - DOCKER_COMPOSE_VERSION=1.11.2
+        - DOCKER_COMPOSE_VERSION=1.18.0
         - PIP_DOWNLOAD_CACHE=".pip_download_cache"
         - SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config.py
 

Dockerfile

@@ -1,5 +1,4 @@
-
-# Copyright 2014 Netflix, Inc.
+# Copyright 2018 Netflix, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,34 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:14.04
+FROM ubuntu:latest
 MAINTAINER Netflix Open Source Development <[email protected]>
 
-ENV SECURITY_MONKEY_VERSION=v0.9.2 \
+ENV SECURITY_MONKEY_VERSION=v1.0 \
     SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-docker.py
 
-RUN apt-get update &&\
-  apt-get -y -q install python-software-properties software-properties-common postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 curl &&\
-  apt-get install -y python-pip python-dev python-psycopg2 libffi-dev libpq-dev libyaml-dev libxml2-dev libxmlsec1-dev git sudo swig &&\
-  rm -rf /var/lib/apt/lists/*
-
-RUN pip install setuptools --upgrade
-RUN pip install pip --upgrade
-RUN pip install "urllib3[secure]" --upgrade
-
-RUN cd /usr/local/src &&\
-#   git clone --branch $SECURITY_MONKEY_VERSION https://github.com/Netflix/security_monkey.git
-  /bin/mkdir -p security_monkey
 ADD . /usr/local/src/security_monkey
 
-RUN cd /usr/local/src/security_monkey &&\
-  python setup.py install &&\
-  /bin/mkdir -p /var/log/security_monkey/
-
-RUN chmod +x /usr/local/src/security_monkey/docker/*.sh &&\
-  mkdir -pv /var/log/security_monkey &&\
-  /usr/bin/touch /var/log/security_monkey/securitymonkey.log
-  # ln -s /dev/stdout /var/log/security_monkey/securitymonkey.log
+SHELL ["/bin/bash", "-c"]
+RUN apt-get update && \
+    apt-get install -y wget build-essential python-pip python-dev python-psycopg2 postgresql postgresql-contrib libpq-dev nginx supervisor git libffi-dev gcc python-virtualenv -y && \
+    cd /usr/local/src/security_monkey && \
+    chown -R www-data /usr/local/src/security_monkey && \
+    virtualenv venv && \
+    source venv/bin/activate && \
+    pip install setuptools --upgrade && \
+    pip install pip --upgrade && \
+    pip install "urllib3[secure]" --upgrade && \
+    pip install google-compute-engine && \
+    pip install cloudaux\[gcp\] && \
+    pip install cloudaux\[openstack\] && \
+    pip install . && \
+    /bin/mkdir -p /var/log/security_monkey/ && \
+    chmod +x /usr/local/src/security_monkey/docker/*.sh && \
+    /usr/bin/touch /var/log/security_monkey/securitymonkey.log
 
 WORKDIR /usr/local/src/security_monkey
 EXPOSE 5000

README.md

@@ -3,27 +3,47 @@ Security Monkey
 
 <img align="right" alt="Security Monkey Logo 2017" src="docs/images/Security_Monkey.png" width="50%">
 
-Security Monkey monitors your [AWS and GCP accounts](https://medium.com/@Netflix_Techblog/netflix-security-monkey-on-google-cloud-platform-gcp-f221604c0cc7) for policy changes and alerts on insecure configurations.  It provides a single UI to browse and search through all of your accounts, regions, and cloud services.  The monkey remembers previous states and can show you exactly what changed, and when.
+Security Monkey monitors your [AWS and GCP accounts](https://medium.com/@Netflix_Techblog/netflix-security-monkey-on-google-cloud-platform-gcp-f221604c0cc7) for policy changes and alerts on insecure configurations.  Support is available for OpenStack public and private clouds.  Security Monkey can also watch and monitor your GitHub organizations, teams, and repositories.
+
+It provides a single UI to browse and search through all of your accounts, regions, and cloud services.  The monkey remembers previous states and can show you exactly what changed, and when.
 
 Security Monkey can be extended with [custom account types](docs/plugins.md), [custom watchers](docs/development.md#adding-a-watcher), [custom auditors](docs/development.md#adding-an-auditor), and [custom alerters](docs/misc.md#custom-alerters).
 
 It works on CPython 2.7. It is known to work on Ubuntu Linux and OS X.
 
-[![Stories in Ready](https://badge.waffle.io/Netflix/security_monkey.svg?label=ready&title=Ready)](http://waffle.io/Netflix/security_monkey) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/Netflix/security_monkey)
+[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/Netflix/security_monkey)
 
 | Develop Branch  | Master Branch |
 | ------------- | ------------- |
 | [![Build Status](https://travis-ci.org/Netflix/security_monkey.svg?branch=develop)](https://travis-ci.org/Netflix/security_monkey)  | [![Build Status](https://travis-ci.org/Netflix/security_monkey.svg?branch=master)](https://travis-ci.org/Netflix/security_monkey)  |
 | [![Coverage Status](https://coveralls.io/repos/github/Netflix/security_monkey/badge.svg?branch=develop)](https://coveralls.io/github/Netflix/security_monkey?branch=develop)  | [![Coverage Status](https://coveralls.io/repos/github/Netflix/security_monkey/badge.svg?branch=master)](https://coveralls.io/github/Netflix/security_monkey?branch=master) |
 
+🚨⚠️🥁🎺 PLEASE READ: BREAKING CHANGES FOR 1.0 🎺🥁⚠️🚨
+--------------
+If you are upgrading to 1.0 for the first time, please review the [Quickstart](docs/quickstart.md) and the [Autostarting](docs/autostarting.md)
+documents as there is a new deployment pattern for Security Monkey. Also, new IAM permissions have been added.
 
 Project resources
 -----------------
 
+- [Security Monkey Architecture](docs/architecture.md)
 - [Quickstart](docs/quickstart.md)
 - [Upgrading](docs/update.md)
 - [Changelog](docs/changelog.md)
 - [Source code](https://github.com/netflix/security_monkey)
 - [Issue tracker](https://github.com/netflix/security_monkey/issues)
 - [Gitter.im Chat Room](https://gitter.im/Netflix/security_monkey)
 - [CloudAux](https://github.com/Netflix-Skunkworks/cloudaux)
+- [PolicyUniverse](https://github.com/Netflix-Skunkworks/policyuniverse)
+- [Troubleshooting](docs/troubleshooting.md)
+
+Instance Diagram
+---------------
+The components that make up Security Monkey are as follows (not AWS specific):
+![diagram](docs/images/sm_instance_diagram.png)
+
+
+Access Diagram
+------------
+Security Monkey accesses accounts to scan via credentials it is provided ("Role Assumption" where available).
+![diagram](docs/images/sm_iam_diagram.png)

celeryconfig.py

@@ -0,0 +1,26 @@
+"""
+.. module: celeryconfig
+    :platform: Unix
+    :synopsis: Use this file to set up the Celery configuration for task scheduling.
+
+.. version:: $$VERSION$$
+.. moduleauthor:: Mike Grima <[email protected]>
+
+"""
+# Broker source: Place yours here:
+broker_url = 'redis://localhost:6379/0'     # Default assume Redis on localhost
+
+# List of modules to import when the Celery worker starts.
+imports = ('security_monkey.task_scheduler.tasks',)
+
+# How many processes per worker instance?
+worker_concurrency = 10
+
+timezone = "UTC"
+enable_utc = True
+
+###########################
+# IMPORTANT: This helps avoid memory leak issues - do not change this number!
+worker_max_tasks_per_child = 1
+############################
+

dart/lib/component/account_view_component/account_view_component.html

@@ -87,16 +87,14 @@ <h1 ng-if="create" class="page-header">Create Account</h1>
             </div>
           </div>
           <hr />
-          <h3 ng-if="create">Due to an open <a href="https://github.com/Netflix/security_monkey/issues/54">issue</a>. You must restart the scheduler after adding a new account.</h3>
-          <pre ng-if="create" class="code">
-$ sudo  supervisorctl
-supervisor> status
-securitymonkeyapi                RUNNING    pid 19198, uptime 0:00:05
+          <h3 ng-if="create">Due to an open <a href="https://github.com/Netflix/security_monkey/issues/54">issue</a>. You must restart the scheduler on the scheduler instance after adding a new account.</h3>
+          <pre ng-if="create" class="code">$ sudo supervisorctl
+supervisor&gt; status
 securitymonkeyscheduler          RUNNING    pid 19199, uptime 0:00:05
-supervisor> restart securitymonkeyscheduler
+supervisor&gt; restart securitymonkeyscheduler
 securitymonkeyscheduler: stopped
 securitymonkeyscheduler: started
-supervisor>
+supervisor&gt;
           </pre>
 
         </form>

dart/lib/component/item_table_component/item_table_component.html

@@ -69,7 +69,7 @@ <h3 class="panel-title">Items</h3>
           <td>{{item.region}}</td>
           <td><a href="#/viewitem/{{item.id}}">{{item.name}}</a></td>
           <td>{{item.number_issues}}</td>
-          <td>{{item.totalScore()}}</td>
+          <td>{{item.unjustifiedScore()}}</td>
           <td>{{item.first_seen| date:'short'}}</td>
           <td>{{item.last_modified| date:'short'}}</td>
         </tr>

dart/lib/component/settings/watcher_config_component/watcher_config_component.html

@@ -1,14 +1,26 @@
 <div>
   <div class="col-lg-12" ng-switch="isLoaded">
     <div class="panel panel-info">
-      <div class="panel-heading">Users <span class="badge pull-right">{{ items_displayed() }} of {{ totalItems }}</span></div>
+      <div class="panel-heading">Watcher Configuration <span class="badge pull-right">{{ items_displayed() }} of {{ totalItems }}</span></div>
       <div class="panel-body" ng-switch-when="false" ng-switch="isError">
         <p ng-switch-when="false">Loading . . .</p>
         <div ng-switch-when="true" class="alert alert-danger">
           {{err_message}}
         </div>
       </div>
       <div class="panel-body" ng-switch-when="true">
+        <div>
+          <h3>Due to an open <a href="https://github.com/Netflix/security_monkey/issues/54">issue</a>. You must restart the scheduler on the scheduler instance after altering a watcher configuration.</h3>
+              <pre class="code">
+$ sudo supervisorctl
+supervisor&gt; status
+securitymonkeyscheduler          RUNNING    pid 19199, uptime 0:00:05
+supervisor&gt; restart securitymonkeyscheduler
+securitymonkeyscheduler: stopped
+securitymonkeyscheduler: started
+supervisor&gt;
+              </pre>
+        </div>
           <table class="table table-striped">
               <tr>
                   <th>Technology</th>

dart/lib/model/Issue.dart

@@ -9,6 +9,7 @@ class Issue {
     int score;
     String issue;
     String notes;
+    bool fixed;
     bool justified;
     String justified_user;
     String justification;
@@ -25,6 +26,7 @@ class Issue {
         score = data['score'];
         issue = data['issue'];
         notes = data['notes'];
+        fixed = data['fixed'];
         justified = data['justified'];
         justified_user = data['justified_user'];
         justification = data['justification'];

dart/lib/model/Item.dart

@@ -64,11 +64,14 @@ class Item {
     if (data.containsKey('issues')) {
       for (var issue in data['issues']) {
         Issue issueObj = new Issue.fromMap(issue);
-        issues.add(issueObj);
-        if (issueObj.justified) {
-          justified_issues.add(issueObj);
-        } else {
-          unjustified_issues.add(issueObj);
+        // Don't display fixed issues
+        if (issueObj.fixed == false) {
+          issues.add(issueObj);
+          if (issueObj.justified) {
+            justified_issues.add(issueObj);
+          } else {
+            unjustified_issues.add(issueObj);
+          }
         }
       }
     }

dart/pubspec.yaml

@@ -1,6 +1,6 @@
 name: security_monkey
 description: An AWS Policy Monitoring and Alerting Tool
-version: 0.9.2
+version: 1.0.0
 dependencies:
   angular: "^1.1.2+2"
   angular_ui: ">=0.6.8 <0.7.0"
@@ -26,4 +26,4 @@ transformers:
     suppressHints: false
     terse: false
 author: yes
-homepage: https://github.com/Netflix/security_monkey
+homepage: https://github.com/Netflix/security_monkey

dart/web/css/main.css

@@ -1,11 +1,17 @@
-body { padding-top: 70px; }
+body {
+    padding-top: 70px;
+    font-family: "Helvetica Neue", Helvetica, Arial, sans-serif !important;
+}
 
 .ng-cloak {
   display: none !important;
 }
 
 .smhead {
     margin-top: 5px; margin-right: 5px;
+    width: 30px;
+    height: 30px;
+    order: 1;
 }
 
 .smstanding {
@@ -19,3 +25,12 @@ body { padding-top: 70px; }
 .black {
     color: black;
 }
+
+.logo_container {
+    display: flex !important;
+    flex-direction: row;
+}
+
+.sm_logo_text {
+    order: 2;
+}

dart/web/js/sso.js

@@ -16,7 +16,19 @@ var create_url = function(provider) {
 
     if (provider.name.toLowerCase() == "onelogin") {
         url += "?return_to="+next;
-    } else { // google/ping
+    } else if (provider.name.toLowerCase() == "aad") { // AzureAD
+        url += "?";
+        url += "response_type="+provider.responseType;
+        url += "&client_id="+provider.clientId;
+        url += "&redirect_uri="+provider.redirectUri;
+        url += "&nonce="+provider.nonce;
+        url += "&scope="+provider.scope.join(provider.scopeDelimiter);
+        url += "&response_mode="+provider.response_mode;
+        url += "&state=clientId,"+provider.clientId+",redirectUri,"+provider.redirectUri+",return_to,"+next;
+        if (provider.hd) {
+            url += "&hd="+provider.hd;
+        }
+    } else { // google || ping
         url += "?";
         url += "response_type="+provider.responseType;
         url += "&client_id="+provider.clientId;
@@ -26,7 +38,6 @@ var create_url = function(provider) {
         if (provider.hd) {
             url += "&hd="+provider.hd;
         }
-
     }
     return url;
 };
@@ -43,4 +54,4 @@ $.getJSON("/api/1/auth/providers",
             });
         });
     }
-);
+);