Merge pull request #122 from rackerlabs/issue-121-replace-nmap-mentio…

…ns-to-scan

Issue 121 replace nmap mentions to scan

README.md

@@ -281,7 +281,7 @@ the same time.  Keep at 1 to avoid a doubling scanning race condition.
 
 **target_files_dir:** Name of actual agent `target_files` directory on the agent box.
 
-**nmap_results_dir:** Name of actual agent `nmap_results` directory on the agent box.
+**scan_results_dir:** Name of actual agent `scan_results` directory on the agent box.
 
 **log_verbosity:** Desired log level for logs/agent.log
 
@@ -384,10 +384,10 @@ jobs.  The username and agent name are the same from the webapp's point of view.
 * Place files with target IPs/hosts (fed to nmap `-iL` switch) in `master/target_files/`
 * `target_files` is an NFS share on Master that the agent reads from through an SSH tunnel.
 
-### Master `nmap_results` folder
+### Master `scan_results` folder
 
 * nmap scan results from agents go here.
-* `master/nmap_results/` is an NFS share on Master that the agent writes to through an SSH tunnel.
+* `master/scan_results/` is an NFS share on Master that the agent writes to through an SSH tunnel.
 
 ### Master Troubleshooting
 
@@ -507,26 +507,26 @@ Source: <https://security.stackexchange.com/questions/78618/is-there-a-nmap-comm
 5. View pending scans
 
     ```bash
-    cd /home/scantron/master/nmap_results/pending
+    cd /home/scantron/master/scan_results/pending
     ls -lart
     ```
 
-    Completed scans are moved to the `/home/scantron/master/nmap_results/completed` directory.
+    Completed scans are moved to the `/home/scantron/master/scan_results/completed` directory.
 
 6. Process scans
 
     Scan files are moved between a few folders.
 
-    `/home/scantron/master/nmap_results/pending` - Pending scan files from agents are stored here before being moved to
-    nmap_results/complete
+    `/home/scantron/master/scan_results/pending` - Pending scan files from agents are stored here before being moved to
+    scan_results/complete
 
-    `/home/scantron/master/nmap_results/complete` - Completed scan files from agents are stored here before being
+    `/home/scantron/master/scan_results/complete` - Completed scan files from agents are stored here before being
     processed by nmap_to_csv.py
 
     The `scantron` user executes a cron job (`nmap_to_csv.sh` which calls `nmap_to_csv.py`) every 5 minutes that will
     process the `.xml` scan results found in the `complete` directory and move them to the `processed` directory.
 
-    `/home/scantron/master/nmap_results/processed` - nmap scan files already processed by nmap_to_csv.py reside here.
+    `/home/scantron/master/scan_results/processed` - nmap scan files already processed by nmap_to_csv.py reside here.
 
     `/home/scantron/master/for_splunk` - csv files for Spulnk ingestion
 

agent/agent.py

@@ -15,7 +15,7 @@
 # Custom Python libraries.
 import modules.api
 import modules.logger
-import modules.nmap_scanner
+import modules.scanner
 
 
 class Worker(threading.Thread):
@@ -35,11 +35,11 @@ def run(self):
 
             try:
                 # Kick off scan.
-                nmap_process = multiprocessing.Process(target=modules.nmap_scanner.scan_site, args=(scan_job_dict,))
-                nmap_process.start()
+                scan_process = multiprocessing.Process(target=modules.scanner.scan_site, args=(scan_job_dict,))
+                scan_process.start()
 
             except Exception as e:
-                modules.logger.ROOT_LOGGER.error("Failed to start scan. Exception: {}".format(e))
+                modules.logger.ROOT_LOGGER.error(f"Failed to start scan.  Exception: {e}")
 
             agent.queue.task_done()
 
@@ -67,7 +67,7 @@ def load_config(self, config_file):
                 return json_data
 
         else:
-            modules.logger.ROOT_LOGGER.error("{} does not exist or contains no data.".format(config_file))
+            modules.logger.ROOT_LOGGER.error(f"'{config_file}' does not exist or contains no data.")
             sys.exit(0)
 
     def go(self):
@@ -82,9 +82,7 @@ def go(self):
             thread.daemon = True
             thread.start()
 
-        modules.logger.ROOT_LOGGER.info(
-            "Starting scan agent: {}".format(self.config_data["scan_agent"]), exc_info=False
-        )
+        modules.logger.ROOT_LOGGER.info(f"Starting scan agent: {self.config_data['scan_agent']}", exc_info=False)
 
         while True:
             try:
@@ -93,7 +91,7 @@ def go(self):
 
                 if scan_jobs:
                     for scan_job in scan_jobs:
-                        modules.logger.ROOT_LOGGER.info("Executing scan job ID: {}".format(scan_job["id"]))
+                        modules.logger.ROOT_LOGGER.info(f"Executing scan job ID: {scan_job['id']}")
 
                         # Create new dictionary that will contain scan_job and config_data information.
                         scan_job_dict = {}
@@ -114,9 +112,7 @@ def go(self):
 
                 else:
                     modules.logger.ROOT_LOGGER.info(
-                        "No scan jobs found...checking back in {} seconds.".format(
-                            self.config_data["callback_interval_in_seconds"]
-                        )
+                        f"No scan jobs found...checking back in {self.config_data['callback_interval_in_seconds']} seconds."
                     )
                     time.sleep(self.config_data["callback_interval_in_seconds"])
 
@@ -127,8 +123,15 @@ def go(self):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Scantron nmap scan agent")
-    parser.add_argument("-c", dest="config_file", action="store", required=True, help="Configuration file.")
+    parser = argparse.ArgumentParser(description="Scantron scan agent")
+    parser.add_argument(
+        "-c",
+        dest="config_file",
+        action="store",
+        required=False,
+        default="agent_config.json",
+        help="Configuration file.  Defaults to 'agent_config.json'",
+    )
     args = parser.parse_args()
 
     config_file = args.config_file

agent/agent_config.json

@@ -6,7 +6,7 @@
     "callback_interval_in_seconds": 60,
     "number_of_threads": 1,
     "target_files_dir": "./target_files",
-    "nmap_results_dir": "./nmap_results",
+    "scan_results_dir": "./scan_results",
     "log_verbosity": 20,
     "http_useragent": "user-agent"
 }

agent/modules/nmap_scanner.py → agent/modules/scanner.py

@@ -12,14 +12,14 @@
 from . import logger
 
 
-def build_masscan_command(nmap_command, target_file, json_file, http_useragent):
+def build_masscan_command(scan_command, target_file, json_file, http_useragent):
     """Builds the masscan command."""
 
     # Can only have 1 file output type.
     file_options = f"-iL {target_file} -oJ {json_file} --http-user-agent {http_useragent}"
 
-    # nmap_command is used for both nmap and masscan commands.
-    masscan_command = f"masscan {nmap_command} {file_options}"
+    # scan_command is used for both nmap and masscan commands.
+    masscan_command = f"masscan {scan_command} {file_options}"
 
     return masscan_command
 
@@ -35,10 +35,10 @@ def scan_site(scan_job_dict):
         # Assign variables.
         site_name = scan_job["site_name"]
         scan_binary = scan_job["scan_binary"]
-        nmap_command = scan_job["nmap_command"]  # Also used for masscan.
+        scan_command = scan_job["scan_command"]  # Also used for masscan.
         result_file_base_name = scan_job["result_file_base_name"]
 
-        nmap_results_dir = config_data["nmap_results_dir"]
+        scan_results_dir = config_data["scan_results_dir"]
 
         target_files_dir = config_data["target_files_dir"]
         target_file = os.path.join(target_files_dir, f"{result_file_base_name}.targets")
@@ -53,8 +53,8 @@ def scan_site(scan_job_dict):
             fh.write(f"{targets}")
 
         # Setup folder structure.
-        pending_files_dir = os.path.join(nmap_results_dir, "pending")
-        complete_files_dir = os.path.join(nmap_results_dir, "complete")
+        pending_files_dir = os.path.join(scan_results_dir, "pending")
+        complete_files_dir = os.path.join(scan_results_dir, "complete")
 
         if scan_binary == "masscan":
             # Output format.
@@ -97,12 +97,12 @@ def scan_site(scan_job_dict):
                     )
 
                     # Build the masscan command.
-                    command = build_masscan_command(nmap_command, target_file, json_file, config_data["http_useragent"])
+                    command = build_masscan_command(scan_command, target_file, json_file, config_data["http_useragent"])
 
             # New scan.
             else:
                 # Build the masscan command.
-                command = build_masscan_command(nmap_command, target_file, json_file, config_data["http_useragent"])
+                command = build_masscan_command(scan_command, target_file, json_file, config_data["http_useragent"])
 
         elif scan_binary == "nmap":
             # Three different nmap scan result file types.
@@ -122,7 +122,7 @@ def scan_site(scan_job_dict):
                 file_options = "-iL {} -oG {} -oN {} -oX {} --script-args http.useragent='{}'".format(
                     target_file, gnmap_file, nmap_file, xml_file, config_data["http_useragent"]
                 )
-                command = "nmap {} {}".format(nmap_command, file_options)
+                command = "nmap {} {}".format(scan_command, file_options)
 
         else:
             logger.ROOT_LOGGER.error("Invalid scan binary specified: {}".format(scan_binary))

agent/nfs_watcher.sh

@@ -2,13 +2,13 @@
 
 # Runs as a cronjob every minute.  If the NFS share is not established, re-mounts it.
 
-nmap_results="$(stat -f -L -c %T /root/agent/nmap_results)"
+scan_results="$(stat -f -L -c %T /root/agent/scan_results)"
 target_files="$(stat -f -L -c %T /root/agent/target_files)"
 
-if [ "$nmap_results" != "nfs" ]
+if [ "$scan_results" != "nfs" ]
 then
-    echo "[+] mounting nmap_results"
-    mount -o rw,hard,noexec,tcp,port=2049 127.0.0.1:/home/scantron/master/nmap_results /root/agent/nmap_results
+    echo "[+] mounting scan_results"
+    mount -o rw,hard,noexec,tcp,port=2049 127.0.0.1:/home/scantron/master/scan_results /root/agent/scan_results
 fi
 
 if [ "$target_files" != "nfs" ]

agent/scan_results/README

@@ -0,0 +1 @@
+NFS share to master that contains scan_results.

ansible-playbooks/roles/agent/tasks/main.yml

@@ -62,7 +62,7 @@
     src: ../../../../agent
     dest: /root
   # Only required if re-running Ansible on an agent that already has comms with master.
-  # Ansible will get angry since it can't chown /root/agent/nmap_results and /root/agent/target_files
+  # Ansible will get angry since it can't chown /root/agent/scan_results and /root/agent/target_files
   ignore_errors: yes
 
 - name: Change owner of autossh.key to autossh.

ansible-playbooks/roles/agent/templates/agent_config.json

@@ -6,7 +6,7 @@
     "callback_interval_in_seconds": 60,
     "number_of_threads": 1,
     "target_files_dir": "./target_files",
-    "nmap_results_dir": "./nmap_results",
+    "scan_results_dir": "./scan_results",
     "log_verbosity": 20,
     "http_useragent": "user-agent"
 }

ansible-playbooks/roles/master/tasks/main.yml

@@ -287,9 +287,9 @@
     group: nogroup
     recurse: yes
 
-- name: Change ownership of nmap_results
+- name: Change ownership of scan_results
   file:
-    path: "/home/{{ non_root_user }}/master/nmap_results"
+    path: "/home/{{ non_root_user }}/master/scan_results"
     owner: nobody
     group: nogroup
     recurse: yes
@@ -321,7 +321,7 @@
 
 - name: Make nmap_to_csv.sh and nmap_to_csv.py executable.
   file:
-    path: "/home/{{ non_root_user }}//master/nmap_results/{{ item }}"
+    path: "/home/{{ non_root_user }}/master/scan_results/{{ item }}"
     owner: root
     group: root
     mode: 0700
@@ -366,7 +366,7 @@
     day: "*"
     month: "*"
     weekday: "*"
-    job: "{{ scantron_dir }}/nmap_results/nmap_to_csv.sh"
+    job: "{{ scantron_dir }}/scan_results/nmap_to_csv.sh"
     user: root
 
 - name: Add crontab entry for masscan_json_to_csv.sh
@@ -377,7 +377,7 @@
     day: "*"
     month: "*"
     weekday: "*"
-    job: "{{ scantron_dir }}/nmap_results/masscan_json_to_csv.sh"
+    job: "{{ scantron_dir }}/scan_results/masscan_json_to_csv.sh"
     user: root
 
 - name: Copying custom logrotate configuration file to /etc/logrotate.d/

ansible-playbooks/roles/master/templates/etc/exports.j2

@@ -12,4 +12,4 @@
 # sync: Forces NFS to write changes to disk before replying.
 # no_subtree_check:  Disable subtree checking
 /home/{{ non_root_user }}/master/target_files    127.0.0.1(rw,sync,no_subtree_check,insecure)
-/home/{{ non_root_user }}/master/nmap_results    127.0.0.1(rw,sync,no_subtree_check,insecure)
+/home/{{ non_root_user }}/master/scan_results    127.0.0.1(rw,sync,no_subtree_check,insecure)

ansible-playbooks/roles/master/templates/etc/nginx/sites-available/scantron_nginx.conf.j2

@@ -31,7 +31,7 @@ server {
     # https://wellfire.co/learn/nginx-django-x-accel-redirects/
     location /protected/complete {
         internal;
-        alias {{ scantron_dir }}/nmap_results/processed;
+        alias {{ scantron_dir }}/scan_results/processed;
     }
 
     location /static {

master/django_scantron/__init__.py

@@ -1 +1 @@
-__version__ = "1.5"
+__version__ = "1.6"

master/django_scantron/admin.py

@@ -9,9 +9,9 @@ class AgentAdmin(admin.ModelAdmin):
     readonly_fields = ("id", "scan_agent", "api_token")
 
 
-class NmapCommandAdmin(admin.ModelAdmin):
+class ScanCommandAdmin(admin.ModelAdmin):
 
-    list_display = ("id", "scan_binary", "nmap_scan_name", "nmap_command")
+    list_display = ("id", "scan_binary", "scan_command_name", "scan_command")
 
 
 class ScanAdmin(admin.ModelAdmin):
@@ -22,7 +22,7 @@ class ScanAdmin(admin.ModelAdmin):
 
 
 class SiteAdmin(admin.ModelAdmin):
-    list_display = ("id", "site_name", "description", "targets", "nmap_command", "scan_agent")
+    list_display = ("id", "site_name", "description", "targets", "scan_command", "scan_agent")
 
 
 class ScheduledScanAdmin(admin.ModelAdmin):
@@ -36,8 +36,8 @@ class ScheduledScanAdmin(admin.ModelAdmin):
         "scan_agent_id",
         "start_datetime",
         "scan_binary",
-        "nmap_command",
-        "nmap_command_id",
+        "scan_command",
+        "scan_command_id",
         "targets",
         "scan_status",
         "completed_time",
@@ -52,7 +52,7 @@ def _register(model, admin_class):
 
 
 _register(models.Agent, AgentAdmin)
-_register(models.NmapCommand, NmapCommandAdmin)
+_register(models.ScanCommand, ScanCommandAdmin)
 _register(models.Scan, ScanAdmin)
 _register(models.ScheduledScan, ScheduledScanAdmin)
 _register(models.Site, SiteAdmin)

master/django_scantron/api/serializers.py

@@ -3,7 +3,7 @@
 # fmt: off
 from django_scantron.models import (
     Agent,
-    NmapCommand,
+    ScanCommand,
     Scan,
     ScheduledScan,
     Site,
@@ -20,14 +20,14 @@ class Meta:
         fields = ("scan_agent", "description", "api_token",)
 
 
-class NmapCommandSerializer(serializers.ModelSerializer):
+class ScanCommandSerializer(serializers.ModelSerializer):
     class Meta:
-        model = NmapCommand
-        fields = ("scan_binary", "nmap_scan_name", "nmap_command",)
+        model = ScanCommand
+        fields = ("scan_binary", "scan_command_name", "scan_command",)
 
 
 class SiteSerializer(serializers.ModelSerializer):
-    nmap_command = serializers.StringRelatedField(many=False)
+    scan_command = serializers.StringRelatedField(many=False)
     scan_agent = serializers.StringRelatedField(many=False)
 
     # Separate validation needed for DRF; doesn't use model's clean() function anymore.
@@ -53,7 +53,7 @@ class Meta:
             "site_name",
             "description",
             "targets",
-            "nmap_command",
+            "scan_command",
             "scan_agent",
         )
 
@@ -77,7 +77,7 @@ class Meta:
             "scan_agent",
             "start_datetime",
             "scan_binary",
-            "nmap_command",
+            "scan_command",
             "targets",
             "scan_status",
             "completed_time",