Merge pull request socallinuxexpo#642 from socallinuxexpo/rh/16994287…

…93monitor

[READY] - monitoring: grafana and prometheus service enabled

facts/inventory.py

@@ -490,6 +490,19 @@ def generatekeaconfig(servers, aps, vlans, outputdir):
         f.write(json.dumps(kea_config, indent=2))
 
 
+def generatepromconfig(servers, aps, vlans, outputdir):
+    prom_config = [
+        {
+            "targets": [ap["ipv4"]],
+            "labels": {"ap": ap["name"]},
+        }
+        for ap in aps
+    ]
+
+    with open(f'{outputdir}/prom.json', 'w') as f:
+        f.write(json.dumps(prom_config, indent=2))
+
+
 def generatezones(switches,routers,pis,aps,servers, outputdir):
     content=''
     for batch in [switches, routers,pis,aps,servers]:
@@ -564,9 +577,12 @@ def main():
         generatekeaconfig(servers,aps,vlans,outputdir)
     elif subcomm == 'nsd':
         generatezones(switches,routers,pis,aps,servers,outputdir)
+    elif subcomm == 'prom':
+        generatepromconfig(servers,aps,vlans,outputdir)
     elif subcomm == 'all':
         generatekeaconfig(servers,aps,vlans,outputdir)
         generatezones(switches,routers,pis,aps,servers,outputdir)
+        generatepromconfig(servers,aps,vlans,outputdir)
 
 
 if __name__ == "__main__":

nix/machines/_common/prometheus.nix

@@ -0,0 +1,20 @@
+{ ... }:
+let
+  port = 9100;
+in
+{
+  networking.firewall.allowedTCPPorts = [ port ];
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = port;
+    enabledCollectors = [
+      "logind"
+      "systemd"
+      "network_route"
+    ];
+    disabledCollectors = [
+      "textfile"
+    ];
+  };
+}

nix/machines/flake-module.nix

@@ -21,6 +21,14 @@ in
         ];
         specialArgs = { inherit inputs; };
       };
+      monitor = lib.nixosSystem {
+        inherit system;
+        modules = [
+          common
+          ./monitor.nix
+        ];
+        specialArgs = { inherit inputs; };
+      };
       massflash = lib.nixosSystem {
         inherit system;
         modules = [

nix/machines/monitor.nix

@@ -0,0 +1,92 @@
+{ config, lib, pkgs, inputs, ... }:
+let
+  hostname = "monitoring.scale.lan";
+in
+{
+  imports =
+    [
+      ./_common/prometheus.nix
+    ];
+
+  # If not present then warning and will be set to latest release during build
+  system.stateVersion = "23.05";
+
+  boot.kernelParams = [ "console=ttyS0" "boot.shell_on_fail" ];
+
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+
+  # TODO: How to handle sudo esculation
+  security.sudo.wheelNeedsPassword = false;
+
+  environment.systemPackages = with pkgs; [
+    vim
+    git
+    bintools
+  ];
+
+  services = {
+    openssh = {
+      enable = true;
+    };
+
+    prometheus = {
+      enable = true;
+      enableReload = true;
+      scrapeConfigs = [
+        {
+          job_name = "prometheus";
+          static_configs = [
+            {
+              targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+              labels = { instance = "localhost"; };
+            }
+          ];
+        }
+        {
+          job_name = "ap";
+          static_configs = builtins.fromJSON (builtins.readFile "${inputs.self.packages.${pkgs.system}.scaleInventory}/config/prom.json");
+        }
+      ];
+    };
+
+    grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3000;
+          domain = "${hostname}";
+        };
+        analytics.reporting_enabled = false;
+      };
+      provision = {
+        # Can use just datasources anymore
+        # https://github.com/NixOS/nixpkgs/blob/41de143fda10e33be0f47eab2bfe08a50f234267/nixos/modules/services/monitoring/grafana.nix#L101-L104
+        datasources.settings.datasources = [
+          {
+            name = "prometheus";
+            type = "prometheus";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+          }
+        ];
+      };
+    };
+
+    nginx = {
+      enable = true;
+      # TODO: TLS enabled
+      # Good example enable TLS, but would like to keep it out of the /nix/store
+      # ref: https://github.com/NixOS/nixpkgs/blob/c6fd903606866634312e40cceb2caee8c0c9243f/nixos/tests/custom-ca.nix#L80
+      virtualHosts."${hostname}" = {
+        default = true;
+        # ACME wont work for us on the private network
+        enableACME = false;
+        locations."/" = {
+          proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}/";
+          proxyWebsockets = true;
+        };
+      };
+    };
+  };
+}

tests/unit/openwrt/golden/ar71xx/root/.ssh/authorized_keys

@@ -2,6 +2,7 @@ ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnRaYbdYsnVqTZNRpXxgK1LlEk9QWa/JwaYAbOZFXiC
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEqPnzsYPKyURdnUpZx1nt9RFQjaz9q7m5wh525Crsho dlang@dlang-mobile
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCtTtJZOyg/9/hbP6IuCyjpA1L0SqMR6wWOU8uJaoa3YlN2sqUkIGne1WYc+4jR+0F2uusDQ1Beb2a9Z0XGxP7nkEIGc5ontC6R/ZUHGf8axz5LXGk9VESR6sMdOjeotSYWwcuj6kPqa0XNXy0nG08dhe8Y+QkjiDQRhjMka4OOmcjMtRAjJyfhROEMpFM18M4Fh3+8j36TatzQQWO6wZ408dQYIc6ShleVfVCvEn5fZ0lm3BRe0UW3wfNs9qupk89VrfUWAEYqvh2uSz9SJBEkGAumreu6ASq7rfPC2DyI60vIT4uaRsqSzfQyT9o1n4v8WmgUKp4kRfZ+T8jWFoUXhj82+2WCCxUlq8D1SRcXDI1OQhHNmH7okorw7TgKJPdM0f96tvgdviH3As6xP/GdnEup8HL0nqKSX8dbRggS9xvmr5SKqGN8QSrclJ+cCsUOWRctgGasf7m+Q6XFNF/8LG6wbqBxxw7TLMLkjVdppHAFoewoBau5cRKGQ++G+BU= dlang@dlang-mobile
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEw39EeFaPgkOHaeV14d/m38YrCrxSycX1dfsPYs6epe jimd@scale
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBt5bQ9fv3vUDOFyYu2k3K2zn0hQmuyHTZF0TRPHt8bX Jeff_J
 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBjjcUJLTENGrV6K/nrPOswcBVMMuS4sLSs0UyTRw8wU87PDUzJz8Ht2SgHqeEQJdRm1+b6iLsx2uKOf+/pU8qE= [email protected]
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICVZ7n1EOezedsbphq5atGtHm11xeGpLZBzEbgV7eZdb Ryan Hamel - SCALE
 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMEiESod7DOT2cmT2QEYjBIrzYqTDnJLld1em3doDROq sarcasticadmin