Merge pull request #126 from rackerlabs/issue-126-small-updates

Issue 126 small updates

README.md

@@ -368,10 +368,10 @@ the current datetime.
 
 ```bash
 # Not using SSH tunnels.
-curl -k -X GET -H 'Authorization: Token <VALID API TOKEN>' https://192.168.1.99:443/api/scheduled_scans?format=json
+curl -k -X GET -H 'Authorization: Token <VALID API TOKEN>' https://192.168.1.99:443/api/scheduled_scans
 
 # Using SSH tunnels.
-curl -k -X GET -H 'Authorization: Token <VALID API TOKEN>' https://127.0.0.1:4430/api/scheduled_scans?format=json
+curl -k -X GET -H 'Authorization: Token <VALID API TOKEN>' https://127.0.0.1:4430/api/scheduled_scans
 ```
 
 You can also log into the webapp using the agent name and password and browse to `/api/?format=json` to view any scan
@@ -391,17 +391,19 @@ jobs.  The username and agent name are the same from the webapp's point of view.
 
 ### Master Troubleshooting
 
-Ensure SSH tunnels setup in `/etc/rc.local` are up.
+1). Ensure SSH tunnels setup in `/etc/rc.local` are up.
 
 ```bash
 netstat -nat | egrep "192.168.1.100|192.168.1.101"
 ps -ef | egrep autossh
 ```
 
-Check nginx logs for agent name in User-Agent field to determine which agents are calling home.
+2). Django logs can be found here: `/var/log/webapp/django_scantron.log`
+
+3). Check nginx logs for agent name in User-Agent field to determine which agents are calling home.
 nginx logs: `tail -f /var/log/nginx/{access,error}.log`
 
-uwsgi logs: `/home/scantron/master/logs`
+4). uwsgi logs: `/home/scantron/master/logs`
 
 ### Known issues with Master NFS share
 

agent/modules/api.py

@@ -20,16 +20,20 @@ def check_for_scan_jobs(config_data):
     """Check for new scans through the API."""
 
     # Build URL to pull new scan jobs.  Server determines jobs based off agent (user) making request.
-    url = "{}:{}/api/scheduled_scans?format=json".format(
-        config_data["master_address"], config_data["master_port"]
-    )
+    master_address = config_data["master_address"]
+    master_port = config_data["master_port"]
+    api_token = config_data["api_token"]
+
+    url = f"{master_address}:{master_port}/api/scheduled_scans"
     logger.ROOT_LOGGER.info("check_for_scans URL: {}".format(url))
 
     # Update User-Agent and add API token.
+    # fmt:off
     headers = {
         "user-agent": config_data["scan_agent"],
-        "Authorization": "Token {}".format(config_data["api_token"]),
+        "Authorization": f"Token {api_token}",
     }
+    # fmt:on
 
     try:
         # Make the HTTP GET request.
@@ -40,53 +44,47 @@ def check_for_scan_jobs(config_data):
             return response.json()
 
         else:
-            logger.ROOT_LOGGER.error(
-                "Could not access {}. HTTP status code: {}".format(
-                    url, response.status_code
-                )
-            )
+            logger.ROOT_LOGGER.error(f"Could not access {url}. HTTP status code: {response.status_code}")
             return None
 
     except Exception as e:
-        logger.ROOT_LOGGER.error(
-            "api.check_for_scans function exception: {}".format(e)
-        )
+        logger.ROOT_LOGGER.error(f"api.check_for_scan_jobs function exception: {e}")
 
 
 def update_scan_information(config_data, scan_job, update_info):
     """Update scan information using a PATCH API request."""
 
+    master_address = config_data["master_address"]
+    master_port = config_data["master_port"]
+    api_token = config_data["api_token"]
+    scan_agent = config_data["scan_agent"]
+    scan_job_id = scan_job["id"]
+
     # Build URL to update scan job.
-    url = "{}:{}/api/scheduled_scans/{}".format(
-        config_data["master_address"], config_data["master_port"], scan_job["id"]
-    )
-    logger.ROOT_LOGGER.info("update_scan_information URL: {}".format(url))
+    url = f"{master_address}:{master_port}/api/scheduled_scans/{scan_job_id}"
+    logger.ROOT_LOGGER.info(f"update_scan_information URL: {url}")
 
-    # Update the User-Agent, add API token, and add Content-Type.
+    # Update the User-Agent, API token, and Content-Type.
+    # fmt:off
     headers = {
-        "user-agent": config_data["scan_agent"],
-        "Authorization": "Token {}".format(config_data["api_token"]),
+        "user-agent": scan_agent,
+        "Authorization": f"Token {api_token}",
         "Content-Type": "application/json",
     }
+    # fmt:on
 
     # Make the HTTP PATCH request.
-    response = requests.patch(
-        url, headers=headers, verify=False, timeout=15, json=update_info
-    )
+    response = requests.patch(url, headers=headers, verify=False, timeout=15, json=update_info)
 
     if response.status_code == 200:
         logger.ROOT_LOGGER.info(
-            "Successfully updated scan information for scan ID {} with data {}".format(
-                scan_job["id"], update_info
-            )
+            f"Successfully updated scan information for scan ID {scan_job_id} with data {update_info}"
         )
         return None
 
     else:
         logger.ROOT_LOGGER.error(
-            "Could not access {} or failed to update scan ID {}. HTTP status code: {}".format(
-                url, scan_job["id"], response.status_code
-            )
+            f"Could not access {url} or failed to update scan ID {scan_job_id}. HTTP status code: {response.status_code}"
         )
-        logger.ROOT_LOGGER.error("Response content: {}".format(response.content))
+        logger.ROOT_LOGGER.error(f"Response content: {response.content}".format())
         return None

ansible-playbooks/roles/master/tasks/main.yml

@@ -151,6 +151,22 @@
 
 # Django
 ########
+- name: Create /var/log/webapp folder.
+  file:
+    path: /var/log/webapp
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Create /var/log/webapp/django_scantron.log file.
+  file:
+    path: /var/log/webapp/django_scantron.log
+    state: touch
+    owner: www-data
+    group: www-data
+    mode: 0660
+
 - name: Update manage.py with local/production environment variable.
   template:
     src: templates/manage.py.j2

ansible-playbooks/roles/master/templates/production.py.j2

@@ -1,6 +1,5 @@
 """
 Production Configuration
-
 """
 # export DJANGO_SETTINGS_MODULE="config.settings.production"
 from .base import *  # noqa
@@ -17,10 +16,8 @@ DEBUG = False
 
 # set this to 60 seconds and then to 518400 when you can prove it works
 # SECURE_HSTS_SECONDS = 60
-# SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
-#     'DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS', default=True)
-# SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
-#     'DJANGO_SECURE_CONTENT_TYPE_NOSNIFF', default=True)
+# SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
+# SECURE_CONTENT_TYPE_NOSNIFF = env.bool("DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True)
 SECURE_BROWSER_XSS_FILTER = True
 SESSION_COOKIE_SECURE = True
 SESSION_COOKIE_HTTPONLY = True
@@ -43,30 +40,33 @@ ALLOWED_HOSTS = [
 
 # LOGGING CONFIGURATION
 # ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
-# A sample logging configuration. The only tangible logging
-# performed by this configuration is to send an email to
-# the site admins on every HTTP 500 error when DEBUG=False.
-# See https://docs.djangoproject.com/en/dev/topics/logging for
-# more details on how to customize your logging configuration.
+# fmt: off
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
-    "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
-    "formatters": {"verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}},
+    "formatters": {
+        "verbose": {
+            "format": "%(asctime)s %(levelname)s %(module)s " "%(process)d %(thread)d %(message)s",
+        },
+    },
     "handlers": {
-        "mail_admins": {
-            "level": "ERROR",
-            "filters": ["require_debug_false"],
-            "class": "django.utils.log.AdminEmailHandler",
+        "file": {
+            "level": "DEBUG",
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "/var/log/webapp/django_scantron.log",
+            "maxBytes": 1024 * 1024 * 15,  # 15MB
+            "backupCount": 10,
+            "formatter": "verbose",
         },
-        "console": {"level": "DEBUG", "class": "logging.StreamHandler", "formatter": "verbose"},
     },
     "loggers": {
-        "django.request": {"handlers": ["mail_admins"], "level": "ERROR", "propagate": True},
-        "django.security.DisallowedHost": {"level": "ERROR", "handlers": ["console", "mail_admins"], "propagate": True},
+        "django": {
+            "handlers": ["file"],
+            "level": "DEBUG",
+            "propagate": True,
+        },
     },
 }
-
+# fmt: on
 # Your production stuff: Below this line define 3rd party library settings
 # ------------------------------------------------------------------------------

master/django_scantron/__init__.py

@@ -1 +1 @@
-__version__ = "1.7"
+__version__ = "1.8"

master/django_scantron/admin.py

@@ -1,7 +1,22 @@
 from django.contrib import admin
+# from django.contrib.auth.decorators import login_required
+from django.contrib.sessions.models import Session
 from . import models
 
 
+# Taken from django-all-auth: https://django-allauth.readthedocs.io/en/latest/advanced.html#admin
+# "require users to login before going to the Django admin site’s login page"
+# admin.site.login = login_required(admin.site.login)
+
+
+# View sessions in Django Admin.
+class SessionAdmin(admin.ModelAdmin):
+    def _session_data(self, obj):
+        return obj.get_decoded()
+
+    list_display = ["session_key", "_session_data", "expire_date"]
+
+
 class AgentAdmin(admin.ModelAdmin):
 
     list_display = ("id", "scan_agent", "description", "api_token", "last_checkin")
@@ -32,6 +47,7 @@ class ScheduledScanAdmin(admin.ModelAdmin):
         "site_name",
         "site_name_id",
         "scan_id",
+        "start_time",
         "scan_agent",
         "scan_agent_id",
         "start_datetime",
@@ -52,6 +68,8 @@ def _register(model, admin_class):
     admin.site.register(model, admin_class)
 
 
+_register(Session, SessionAdmin)
+
 _register(models.Agent, AgentAdmin)
 _register(models.ScanCommand, ScanCommandAdmin)
 _register(models.Scan, ScanAdmin)

master/django_scantron/models.py

@@ -203,6 +203,7 @@ class ScheduledScan(models.Model):
     scan_id = models.IntegerField(
         validators=[MinValueValidator(1, message="Scan ID must be greater than 0")], verbose_name="Scan ID"
     )
+    start_time = models.TimeField(verbose_name="Scan start time")
     scan_agent = models.CharField(
         unique=False,
         max_length=255,

master/django_scantron/templates/django_scantron/scheduled_scan_list.html

@@ -33,7 +33,7 @@ <h3>
             <td>{{ scan.id }}</td>
             <td>{{ scan.site_name }}</td>
             <td>{{ scan.scan_agent }}</td>
-            <td>{{ scan.start_time|date:"Y-m-j H:i:s" }}</td>
+            <td>{{ scan.start_time|date:"H:i" }}</td>
             <td>{{ scan.scan_binary }}</td>
             <td>{{ scan.scan_binary }} {{ scan.scan_command }}</td>
             <td>{{ scan.scan_status }}</td>
@@ -45,8 +45,10 @@ <h3>
                     <a href="{% url 'retrieve_scan_file' scan.id %}?file_type=xml">{{ scan.result_file_base_name }}.xml</a>
                 </td>
                 {% else %}
-                <td>{{ scan.result_file_base_name }}.nmap</td>
-                <td>{{ scan.result_file_base_name }}.xml</td>
+                <td>
+                    {{ scan.result_file_base_name }}.nmap<br>
+                    {{ scan.result_file_base_name }}.xml
+                </td>
                 {% endif %}
             {% elif scan.scan_binary == "masscan" %}
                 {% if scan.scan_status == "completed" %}

master/scan_scheduler.py

@@ -118,6 +118,7 @@ def main():
             "site_name": site_name,
             "site_name_id": site_name_id,  # Can delete in future.
             "scan_id": scan_id,  # Can delete in future.
+            "start_time": scan_start_time,
             "scan_agent": scan_agent,
             "scan_agent_id": scan_agent_id,  # Can delete in future.
             "start_datetime": start_datetime,